<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"><title>【尚硅谷】用户画像日志记录 | 恒星同学</title><meta name="keywords" content="SpringBoot,Angular"><meta name="author" content="hengxingstu"><meta name="copyright" content="hengxingstu"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta name="description" content="尚硅谷用户画像配置日志记录">
<meta property="og:type" content="article">
<meta property="og:title" content="【尚硅谷】用户画像日志记录">
<meta property="og:url" content="https://hengxingstu.gitee.io/2022/04/24/ITguiguUserProfile/index.html">
<meta property="og:site_name" content="恒星同学">
<meta property="og:description" content="尚硅谷用户画像配置日志记录">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://w.wallhaven.cc/full/e7/wallhaven-e7g71w.jpg">
<meta property="article:published_time" content="2022-04-24T01:25:00.000Z">
<meta property="article:modified_time" content="2022-06-12T08:48:45.424Z">
<meta property="article:author" content="hengxingstu">
<meta property="article:tag" content="大数据">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://w.wallhaven.cc/full/e7/wallhaven-e7g71w.jpg"><link rel="shortcut icon" href="/img/favicon.png"><link rel="canonical" href="https://hengxingstu.gitee.io/2022/04/24/ITguiguUserProfile/"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@6/css/all.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox.css" media="print" onload="this.media='all'"><script>const GLOBAL_CONFIG = { 
  root: '/',
  algolia: undefined,
  localSearch: {"path":"search.xml","languages":{"hits_empty":"找不到您查询的内容：${query}"}},
  translate: undefined,
  noticeOutdate: undefined,
  highlight: {"plugin":"prismjs","highlightCopy":true,"highlightLang":true,"highlightHeightLimit":false},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: false,
    post: false
  },
  runtime: '',
  date_suffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: undefined,
  lightbox: 'fancybox',
  Snackbar: undefined,
  source: {
    justifiedGallery: {
      js: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery@2/dist/fjGallery.min.js',
      css: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery@2/dist/fjGallery.min.css'
    }
  },
  isPhotoFigcaption: false,
  islazyload: false,
  isAnchor: false
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
  title: '【尚硅谷】用户画像日志记录',
  isPost: true,
  isHome: false,
  isHighlightShrink: false,
  isToc: true,
  postUpdate: '2022-06-12 16:48:45'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(win=>{
    win.saveToLocal = {
      set: function setWithExpiry(key, value, ttl) {
        if (ttl === 0) return
        const now = new Date()
        const expiryDay = ttl * 86400000
        const item = {
          value: value,
          expiry: now.getTime() + expiryDay,
        }
        localStorage.setItem(key, JSON.stringify(item))
      },

      get: function getWithExpiry(key) {
        const itemStr = localStorage.getItem(key)

        if (!itemStr) {
          return undefined
        }
        const item = JSON.parse(itemStr)
        const now = new Date()

        if (now.getTime() > item.expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return item.value
      }
    }
  
    win.getScript = url => new Promise((resolve, reject) => {
      const script = document.createElement('script')
      script.src = url
      script.async = true
      script.onerror = reject
      script.onload = script.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        script.onload = script.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(script)
    })
  
      win.activateDarkMode = function () {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = function () {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
          if (t === 'dark') activateDarkMode()
          else if (t === 'light') activateLightMode()
        
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        if (asideStatus === 'hide') {
          document.documentElement.classList.add('hide-aside')
        } else {
          document.documentElement.classList.remove('hide-aside')
        }
      }
    
    const detectApple = () => {
      if(/iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)){
        document.documentElement.classList.add('apple')
      }
    }
    detectApple()
    })(window)</script><meta name="generator" content="Hexo 6.1.0"></head><body><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img is-center"><img src="https://wallpapercave.com/uwp/uwp2365506.jpeg" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="site-data is-center"><div class="data-item"><a href="/archives/"><div class="headline">文章</div><div class="length-num">36</div></a></div><div class="data-item"><a href="/tags/"><div class="headline">标签</div><div class="length-num">23</div></a></div><div class="data-item"><a href="/categories/"><div class="headline">分类</div><div class="length-num">8</div></a></div></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 归档</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page group" href="javascript:void(0);"><i class="fa-fw fas fa-list"></i><span> 娱乐</span><i class="fas fa-chevron-down"></i></a><ul class="menus_item_child"><li><a class="site-page child" href="/gallery/"><i class="fa-fw fas fa-images"></i><span> 图库</span></a></li></ul></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/contact/"><i class="fa-fw fa-fw fas fa-comment-dots"></i><span> 留言板</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div></div></div><div class="post" id="body-wrap"><header class="post-bg" id="page-header" style="background-image: url('https://w.wallhaven.cc/full/e7/wallhaven-e7g71w.jpg')"><nav id="nav"><span id="blog_name"><a id="site-name" href="/">恒星同学</a></span><div id="menus"><div id="search-button"><a class="site-page social-icon search"><i class="fas fa-search fa-fw"></i><span> 搜索</span></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 归档</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page group" href="javascript:void(0);"><i class="fa-fw fas fa-list"></i><span> 娱乐</span><i class="fas fa-chevron-down"></i></a><ul class="menus_item_child"><li><a class="site-page child" href="/gallery/"><i class="fa-fw fas fa-images"></i><span> 图库</span></a></li></ul></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/contact/"><i class="fa-fw fa-fw fas fa-comment-dots"></i><span> 留言板</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div><div id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></div></div></nav><div id="post-info"><h1 class="post-title">【尚硅谷】用户画像日志记录</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="far fa-calendar-alt fa-fw post-meta-icon"></i><span class="post-meta-label">发表于</span><time class="post-meta-date-created" datetime="2022-04-24T01:25:00.000Z" title="发表于 2022-04-24 09:25:00">2022-04-24</time><span class="post-meta-separator">|</span><i class="fas fa-history fa-fw post-meta-icon"></i><span class="post-meta-label">更新于</span><time class="post-meta-date-updated" datetime="2022-06-12T08:48:45.424Z" title="更新于 2022-06-12 16:48:45">2022-06-12</time></span><span class="post-meta-categories"><span class="post-meta-separator">|</span><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/categories/Markdown/">Markdown</a></span></div><div class="meta-secondline"><span class="post-meta-separator">|</span><span class="post-meta-pv-cv" id="" data-flag-title="【尚硅谷】用户画像日志记录"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">阅读量:</span><span id="busuanzi_value_page_pv"></span></span><span class="post-meta-separator">|</span><span class="post-meta-commentcount"><i class="far fa-comments fa-fw post-meta-icon"></i><span class="post-meta-label">评论数:</span><a href="/2022/04/24/ITguiguUserProfile/#post-comment"><span id="twikoo-count"></span></a></span></div></div></div></header><main class="layout" id="content-inner"><div id="post"><article class="post-content" id="article-container"><h1 id="环境配置"><a href="#环境配置" class="headerlink" title="环境配置"></a>环境配置</h1><h2 id="系统准备"><a href="#系统准备" class="headerlink" title="系统准备"></a>系统准备</h2><p>Ubuntu16.04</p>
<p><img src="https://s2.loli.net/2022/05/01/6TJx48l2XBd1Lsc.png" alt="image-20220404141952340"></p>
<h2 id="Hadoop安装"><a href="#Hadoop安装" class="headerlink" title="Hadoop安装"></a>Hadoop安装</h2><p>伪分布式配置</p>
<p>以Hadoop2.7.5版本为例</p>
<p>准备好安装文件</p>
<p><img src="https://s2.loli.net/2022/03/29/NCtIg8ao7cO3Ehw.jpg" alt="img"></p>
<p><img src="https://s2.loli.net/2022/03/29/GpjSbDFnZM8mzAv.jpg" alt="img"></p>
<p>解压安装包</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">tar -zxvf hadoop-2.7.5.tar.gz<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>执行完后，在当前的目录下会出现一个名为 hadoop-2.7.5 的文件目录，这个目录就是 hadoop 的安装目录。</p>
<p>在 Hadoop 中包含了 3 个技术组件，分别是：</p>
<ul>
<li><p>分布式存储技术 - HDFS</p>
</li>
<li><p>分布式资源管理技术 - Yarn</p>
</li>
<li><p>分布式计算技术 - MapReduce</p>
</li>
</ul>
<p>那么，我们常说的 Hadoop 安装，实际上就包含了 HDFS 的安装、Yarn 的安装以及 MapReduce 的安装，我们接下来分别配置安装。</p>
<p>【JDK安装】</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">sudo apt-get install openjdk-8-jdk<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>检查jdk版本，是否安装成功</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">java -version<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p><img src="https://s2.loli.net/2022/03/29/8AD7Um9kwiNCLBR.jpg" alt="img"></p>
<p>成功后是这样的。</p>
<p>查看javahome的位置</p>
<p><img src="https://s2.loli.net/2022/03/29/qZH9oYPwVlJ7EBI.jpg" alt="img"></p>
<p>把Hadoop放到&#x2F;usr&#x2F;local下，并创建软连接</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">sudo mv hadoop-2.7.5 &#x2F;usr&#x2F;local

sudo ln -snf &#x2F;usr&#x2F;local&#x2F;hadoop-2.7.5&#x2F; &#x2F;usr&#x2F;local&#x2F;hadoop<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<p>给Hadoop文件夹赋予权限（生产环境不推荐）</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">chmod -R 777 hadoop-2.7.5&#x2F;<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>配置环境变量</p>
<ul>
<li><p>局部环境变量：~&#x2F;.bashrc</p>
</li>
<li><p>全局环境变量：&#x2F;etc&#x2F;profile</p>
</li>
</ul>
<p>这里我图方便配置全局了</p>
<pre class="line-numbers language-sh" data-language="sh"><code class="language-sh">export HADOOP_HOME&#x3D;&#x2F;usr&#x2F;local&#x2F;hadoop

export PATH&#x3D;$PATH:$HADOOP_HOME&#x2F;bin:$HADOOP_HOME&#x2F;sbin:.&#x2F;:$JAVA_HOME&#x2F;bin

#JAVA_HOME

export JAVA_HOME&#x3D;&#x2F;usr&#x2F;lib&#x2F;jvm&#x2F;java-8-openjdk-amd64

export JRE_HOME&#x3D;$JAVA_HOME&#x2F;jre

export CLASSPATH&#x3D;$JAVA_HOME&#x2F;lib<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>



<p>配置后生效一下</p>
<p>source &#x2F;etc&#x2F;profile</p>
<p>输入Hadoop的命令，你将看到</p>
<p>这时Hadoop安装好了，现在配置伪分布式：</p>
<p>这是它的配置文件以及路径</p>
<img src="https://s2.loli.net/2022/03/30/U76RD1ZkrQdjVHJ.png" alt="image-20220330154053686" style="zoom:80%;" />

<ol>
<li>配置Hadoop运行环境</li>
</ol>
<p>hadoop_env.sh</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell"># The java implementation to use.  
#这里最好直接写绝对路径了，搞不清楚怎么在这里写变量，这样写最保险
export JAVA_HOME&#x3D;&#x2F;usr&#x2F;lib&#x2F;jvm&#x2F;java-8-openjdk-amd64&#x2F;  <span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<ol start="2">
<li>core-site.xml</li>
</ol>
<pre class="line-numbers language-xml" data-language="xml"><code class="language-xml"><span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>configuration</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>fs.defaultFS<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>hdfs://hengxing:9000<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>hadoop.tmp.dir<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>/data/hadoopdata<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>configuration</span><span class="token punctuation">></span></span>  <span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<ol start="3">
<li>hdfs-site.xml</li>
</ol>
<pre class="line-numbers language-xml" data-language="xml"><code class="language-xml"><span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.namenode.name.dir<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>/data/hadoopdata/name<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>为了保证元数据的安全一般配置多个不同目录<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.datanode.data.dir<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>/data/hadoopdata/data<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>datanode 的数据存储目录<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.replication<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>2<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>HDFS 的数据块的副本存储个数, 默认是3<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span> <span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<ol start="4">
<li>mapred-site.xml</li>
</ol>
<p>在hadoop的相关目录中没有此文件，但是有一个mapred-site.xml.template文件，将该文件复制一份为mapred-site.xml</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">cp mapred-site.xml.template mapred-site.xml<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<pre class="line-numbers language-xml" data-language="xml"><code class="language-xml"><span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>configuration</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>mapreduce.framework.name<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>yarn<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>configuration</span><span class="token punctuation">></span></span>  <span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<ol start="5">
<li>yarn-site.xml</li>
</ol>
<pre class="line-numbers language-xml" data-language="xml"><code class="language-xml"><span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>configuration</span><span class="token punctuation">></span></span>        <span class="token comment">&lt;!-- 指定ResourceManager的地址--></span>           <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.resourcemanager.hostname<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>hengxing<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
    <span class="token comment">&lt;!-- 指定reducer获取数据的方式--></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.nodemanager.aux-services<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>mapreduce_shuffle<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.nodemanager.local-dirs<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>file:///data/hadoopdata/yarn/nodemanager<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>configuration</span><span class="token punctuation">></span></span>  <span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<ol start="6">
<li>创建相关目录</li>
</ol>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">sudo mkdir -p &#x2F;data&#x2F;hadoopdata&#x2F;name

sudo mkdir -p &#x2F;data&#x2F;hadoopdata&#x2F;data

sudo mkdir -p &#x2F;data&#x2F;hadoopdata&#x2F;yarn&#x2F;nodemanager
<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>同时给予权限<code>sudo chmod -R 777 /data/</code></p>
<ol start="7">
<li>对HDFS集群进行格式化，HDFS集群是用来存储数据的。</li>
</ol>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hdfs namenode -format<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>看到下图内容说明成功 <img src="https://s2.loli.net/2022/03/30/Ai47ENBrP3Rl6fF.png" alt="image-20220330160138305"></p>
<p>启动集群</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hadoop-daemon.sh start namenode #启动主节点
hadoop-daemon.sh start datanode #启动从节点
yarn-daemon.sh start resourcemanager
yarn-daemon.sh start nodemanager<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span></span></code></pre>

<ol>
<li>启动HDFS集群</li>
</ol>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hadoop-daemon.sh start namenode #启动主节点
hadoop-daemon.sh start datanode #启动从节点<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>

<p><img src="https://s2.loli.net/2022/03/29/s8gKjXZhbcSGnqA.jpg" alt="img"></p>
<ol start="2">
<li>启动YARN集群</li>
</ol>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">yarn-daemon.sh start resourcemanager
yarn-daemon.sh start nodemanager<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>

<p><img src="https://s2.loli.net/2022/03/30/KGPnXZraIL1TMkw.png" alt="image-20220330160554080"></p>
<p><img src="https://s2.loli.net/2022/03/30/aAOcXnigJWzCvKw.png" alt="image-20220330160606652"></p>
<ol start="3">
<li>启动作业历史服务器</li>
</ol>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">mr-jobhistory-daemon.sh start historyserver<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p><img src="https://s2.loli.net/2022/03/29/flKV6z5o2uAdZUx.jpg" alt="img"></p>
<p>jps可以查看是否启动成功</p>
<p>也可以在浏览器的ui界面查看，网址见截图</p>
<p><img src="https://s2.loli.net/2022/03/29/NxleSpOs8EK7M2G.jpg" alt="img"></p>
<p><img src="https://s2.loli.net/2022/03/29/TQelB4fsNnKAhXL.jpg" alt="img"></p>
<p>用自带的圆周率程序测试一下</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">yarn jar &#x2F;usr&#x2F;local&#x2F;hadoop&#x2F;share&#x2F;hadoop&#x2F;mapreduce&#x2F;hadoop-mapreduce-examples-2.7.5.jar pi 4 100<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p><img src="https://s2.loli.net/2022/03/29/C1mM8Diwpy6OToY.jpg" alt="img"></p>
<p>关闭命令</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hadoop-daemon.sh stop namenode
hadoop-daemon.sh stop datanode
yarn-daemon.sh stop resourcemanager
yarn-daemon.sh stop nodemanager<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span></span></code></pre>

<p>关闭节点安全模式</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hdfs dfsadmin -safemode leave<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>遇到[There are 2 missing blocks. The following files may be corrupted:]</p>
<p><a target="_blank" rel="noopener" href="https://stackoverflow.com/questions/19205057/how-to-fix-corrupt-hdfs-files">外网上的解决方式</a> &lt;–  点击即可进入 </p>
<p>找到中断文件</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hengxing@hengxing:~$ hdfs fsck -list-corruptfileblocks
Connecting to namenode via http:&#x2F;&#x2F;hengxing:50070&#x2F;fsck?ugi&#x3D;hengxing&amp;listcorruptfileblocks&#x3D;1&amp;path&#x3D;%2F
The list of corrupt files under path &#39;&#x2F;&#39; are:
blk_1073741838	&#x2F;tmp&#x2F;hadoop-yarn&#x2F;staging&#x2F;history&#x2F;done&#x2F;2022&#x2F;03&#x2F;24&#x2F;000000&#x2F;job_1648112897030_0001-1648113971406-hengxing-QuasiMonteCarlo-1648114019450-4-1-SUCCEEDED-default-1648113983899.jhist
blk_1073741839	&#x2F;tmp&#x2F;hadoop-yarn&#x2F;staging&#x2F;history&#x2F;done&#x2F;2022&#x2F;03&#x2F;24&#x2F;000000&#x2F;job_1648112897030_0001_conf.xml
The filesystem under path &#39;&#x2F;&#39; has 2 CORRUPT files<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>删除它</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hdfs fsck %你文件的路径% -delete<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<h3 id="Hadoop-跑MapReduce-Job-的时候卡住-unhealthy-nodes-x3D-1"><a href="#Hadoop-跑MapReduce-Job-的时候卡住-unhealthy-nodes-x3D-1" class="headerlink" title="Hadoop 跑MapReduce Job 的时候卡住,unhealthy-nodes&#x3D;1"></a>Hadoop 跑MapReduce Job 的时候卡住,unhealthy-nodes&#x3D;1</h3><pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hengxing@hengxing:~$ yarn node -list -all
22&#x2F;03&#x2F;30 22:38:39 INFO client.RMProxy: Connecting to ResourceManager at hengxing&#x2F;192.168.59.140:8032
Total Nodes:1
         Node-Id	     Node-State	Node-Http-Address	Number-of-Running-Containers
  hengxing:40085	      UNHEALTHY	    hengxing:8042	                           0
hengxing@hengxing:~$ <span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>节点被判断为unhealthy，有以下几点原因：</p>
<ul>
<li>节点的磁盘空间占用达到了阈值</li>
<li>节点的文件存储位置没有权限创建文件</li>
</ul>
<p>通常可以用以下几种方式解决</p>
<ul>
<li>提高阈值、清理磁盘空间</li>
<li>给hadoop存文件的文件夹修改权限和拥有者、直接重新设置存储文件夹（记得别放在home文件夹下，会有权限问题）</li>
</ul>
<p>你可以看到报错里说</p>
<p>local-dir are bad，说明这个文件有问题，应该是没权限新建job文件，被判定为了不健康</p>
<h2 id="Hive安装"><a href="#Hive安装" class="headerlink" title="Hive安装"></a>Hive安装</h2><h3 id="安装"><a href="#安装" class="headerlink" title="安装"></a>安装</h3><p>下载解压，加软连接</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">sudo tar xvfz apache-hive-2.3.7-bin.tar.gz -C &#x2F;usr&#x2F;local&#x2F;
sudo ln -snf apache-hive-2.3.7-bin&#x2F; hive<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>

<p>添加环境变量</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">#Hive Home
export HIVE_HOME&#x3D;&#x2F;usr&#x2F;local&#x2F;hive
export PATH&#x3D;$PATH:$HIVE_HOME&#x2F;bin
export CLASSPATH&#x3D;$CLASSPATH:&#x2F;usr&#x2F;local&#x2F;hadoop&#x2F;lib&#x2F;*:.:&#x2F;usr&#x2F;local&#x2F;hive&#x2F;lib&#x2F;*<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span></span></code></pre>

<p>定义仓库目录</p>
<pre class="line-numbers language-none"><code class="language-none">hdfs dfs -mkdir -p &#x2F;user&#x2F;hengxing&#x2F;warehouse
#修改读写权限：
&#x2F;usr&#x2F;local$ hdfs dfs -chmod 777 &#x2F;user&#x2F;hengxing&#x2F;warehouse<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<p>配置文件</p>
<p>新建conf下的hive-site.xml，注意配置文件中的<value></value>中的内容必须在同一行上且不能有空格，即不能包含换行符和空格，否则就会出错，请检查其它的配置是否还在存在这类问题。</p>
<pre class="line-numbers language-xml" data-language="xml"><code class="language-xml"><span class="token prolog">&lt;?xml version="1.0" encoding="UTF-8" standalone="no"?></span>
<span class="token prolog">&lt;?xml-stylesheet type="text/xsl" href="configuration.xsl"?></span><span class="token comment">&lt;!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
--></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>configuration</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>javax.jdo.option.ConnectionURL<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>jdbc:mysql://localhost:3306/metastore?createDatabaseIfNotExist=true<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>
JDBC connect string for a JDBC metastore.
To use SSL to encrypt/authenticate the connection, provide database-specific SSL flag in the connection URL.
For example, jdbc:postgresql://myhost/db?ssl=true for postgres database.
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>hive.metastore.warehouse.dir<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>hdfs://hengxing:9000/user/hengxing/warehouse<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>location of default database for the warehouse<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token comment">&lt;!-- 
    &lt;property>
        &lt;name>hive.metastore.local&lt;/name>
        &lt;value>true&lt;/value>
    &lt;/property>
    在0.10 0.11或者之后的hive版本 hive.metastore.local 属性不再使用,请去掉 --></span>

    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>javax.jdo.option.ConnectionDriverName<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>com.mysql.jdbc.Driver<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>Driver class name for a JDBC metastore<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>javax.jdo.option.ConnectionUserName<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>root<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>javax.jdo.option.ConnectionPassword<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>123456<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>javax.jdo.PersistenceManagerFactoryClass<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
      <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>org.datanucleus.api.jdo.JDOPersistenceManagerFactory<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>class implementing the jdo persistence<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>hive.metastore.uris<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>thrift://hengxing:9083<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>configuration</span><span class="token punctuation">></span></span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>
<p>另外在这里注意一点，这个文件中的<code>hive.metastore.local</code>配置在0.10 0.11或者之后的hive版本 hive.metastore.local 属性不再使用，请删掉，否则会报如下错误：</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">WARN conf.HiveConf: HiveConf of name hive.metastore.local does not exist<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>



<p>把jdbc依赖包导入到hive&#x2F;lib下</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hengxing@hengxing:&#x2F;usr&#x2F;local&#x2F;hive&#x2F;lib$ cp ~&#x2F;pkg&#x2F;mysql-connector-java-5.1.49&#x2F;mysql-connector-java-5.1.49.jar .&#x2F;<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>初始化hive元数据</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hengxing@hengxing:&#x2F;usr&#x2F;local&#x2F;hive$ .&#x2F;bin&#x2F;schematool -dbType mysql -initSchema root 123456<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>成功后返回</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:&#x2F;usr&#x2F;local&#x2F;apache-hive-2.3.7-bin&#x2F;lib&#x2F;log4j-slf4j-impl-2.6.2.jar!&#x2F;org&#x2F;slf4j&#x2F;impl&#x2F;StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:&#x2F;usr&#x2F;local&#x2F;hadoop-2.7.5&#x2F;share&#x2F;hadoop&#x2F;common&#x2F;lib&#x2F;slf4j-log4j12-1.7.10.jar!&#x2F;org&#x2F;slf4j&#x2F;impl&#x2F;StaticLoggerBinder.class]
SLF4J: See http:&#x2F;&#x2F;www.slf4j.org&#x2F;codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
Metastore connection URL:	 jdbc:mysql:&#x2F;&#x2F;localhost:3306&#x2F;metastore?createDatabaseIfNotExist&#x3D;true
Metastore Connection Driver :	 com.mysql.jdbc.Driver
Metastore connection User:	 root
Starting metastore schema initialization to 2.3.0
Initialization script hive-schema-2.3.0.mysql.sql
Initialization script completed
schemaTool completed<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>也可以在MySQL中看到新创建的数据库</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">mysql&gt; show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| metastore          |
| mysql              |
| performance_schema |
| sys                |
+--------------------+
5 rows in set (0.02 sec)<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<h3 id="启动hive元存储thrift服务器"><a href="#启动hive元存储thrift服务器" class="headerlink" title="启动hive元存储thrift服务器"></a>启动hive元存储thrift服务器</h3><pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hengxing@hengxing:~&#x2F;pkg$ hive --service metastore &amp;
[1] 64323<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>
<p>hive打开DEBUG模式</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hive --hiveconf hive.root.logger&#x3D;DEBUG,console<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>为root用户授权</p>
<pre class="line-numbers language-mysql" data-language="mysql"><code class="language-mysql">mysql&gt; grant all privileges on *.* to &#39;root&#39;@&#39;%&#39; identified by &#39;123456&#39; with grant option;
mysql&gt; flush privileges;<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>

<p>mysql警告</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">Wed Mar 30 01:15:25 CST 2022 WARN: Establishing SSL connection without server&#39;s identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn&#39;t set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to &#39;false&#39;. You need either to explicitly disable SSL by setting useSSL&#x3D;false, or set useSSL&#x3D;true and provide truststore for server certificate verification.<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>这是因为MySQL在高版本需要指明是否进行SSL连接，在mysql连接字符串url中加入ssl&#x3D;true或者false即可</p>
<pre class="line-numbers language-xml" data-language="xml"><code class="language-xml">jdbc:mysql://127.0.0.1:3306/framework?characterEncoding=utf8<span class="token entity named-entity" title="&amp;">&amp;amp;</span>useSSL=true<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>之后可能会报错</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">javax.net.ssl.SSLHandshakeException: java.security.cert.CertPathValidatorException: Path does not chain with any of the trust anchors<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>则可以关掉SSL连接<code>useSSL=false</code></p>
<h3 id="HIVE无法正常启动，报找不到jdbc的驱动类"><a href="#HIVE无法正常启动，报找不到jdbc的驱动类" class="headerlink" title="HIVE无法正常启动，报找不到jdbc的驱动类"></a>HIVE无法正常启动，报找不到jdbc的驱动类</h3><p>通过启动脚本：</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">cd $HIVE_HOME

.&#x2F;bin&#x2F;hive  --hiveconf hive.root.logger&#x3D;DEBUG,console<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<p>启动报如下错误：</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">16&#x2F;08&#x2F;25 19:06:18 [main]: DEBUG bonecp.BoneCPDataSource: JDBC URL &#x3D; jdbc:mysql:&#x2F;&#x2F;slave1:3306&#x2F;hive?createDatabaseIfNotExist&#x3D;true, Username &#x3D; hive, partitions &#x3D; 1, max (per partition) &#x3D; 10, min (per partition) &#x3D; 0, idle max age &#x3D; 60 min, idle test period &#x3D; 240 min, strategy &#x3D; DEFAULT

16&#x2F;08&#x2F;25 19:06:19 [BoneCP-pool-watch-thread]: ERROR bonecp.BoneCP: Failed to acquire connection to jdbc:mysql:&#x2F;&#x2F;slave1:3306&#x2F;hive?createDatabaseIfNotExist&#x3D;true. Sleeping for 7000 ms. Attempts left: 5<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<p>这是报没有找到连接mysql的jdbc驱动，&#x3D;&#x3D;但是实际上$HIVE_HOME&#x2F;lib下已经有mysql的驱动包了&#x3D;&#x3D;</p>
<p>说明驱动包没有问题，经过百度后发现是配置文件$HIVE_HOME&#x2F;conf&#x2F;hive-site.xml内配置有问题：</p>
<blockquote>
<p>原因是配置文件中的<value></value>中的内容必须在同一行上且不能有空格，即不能包含换行符和空格，否则就会出错，请检查其它的配置是否还在存在这类问题。</p>
</blockquote>
<p>上一个问题解决了，接着又报下面的错误</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Got exception: java.net.ConnectException Call From hengxing&#x2F;192.168.59.140 to localhost:9000 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http:&#x2F;&#x2F;wiki.apache.org&#x2F;hadoop&#x2F;ConnectionRefused)<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>网上查相关资料的话，会说这是因为hdfs的datanode没启动，但我的明显不是这样，仔细看报错，他说<code>From hengxing/192.168.59.140 to localhost:9000</code>的连接失败，<code>localhost:9000</code>可用吗？我们telnet一下</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hengxing@hengxing:&#x2F;usr&#x2F;local&#x2F;hadoop&#x2F;etc&#x2F;hadoop$ telnet localhost 9000
Trying 127.0.0.1...
telnet: Unable to connect to remote host: Connection refused<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<p>并不行，仔细想想这是哪里的配置，哦，是元数据服务的地址。我应该设置的是hdfs的地址，hdfs的地址又是什么呢？看看你Hadoop的webUI</p>
<p><img src="https://s2.loli.net/2022/03/30/dehg6lTj8X2GbDO.png" alt="image-20220330165909687"></p>
<p>是<code>hengxing:9000</code>，telnet一下</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hengxing@hengxing:&#x2F;usr&#x2F;local&#x2F;hadoop&#x2F;etc&#x2F;hadoop$ telnet hengxing 9000
Trying 192.168.59.140...
Connected to hengxing.
Escape character is &#39;^]&#39;.<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span></span></code></pre>

<p>你看，hdfs是启动的，把hive-site.xml中的<code>hive.metastore.warehouse.dir</code>配置变成这个地址吧</p>
<pre class="line-numbers language-xml" data-language="xml"><code class="language-xml"><span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
	<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>hive.metastore.warehouse.dir<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
	<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>hdfs://hengxing:9000/user/hengxing/warehouse<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
	<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>location of default database for the warehouse<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>但是，问题来了，按理说这两个地址都是我当前的本机啊，并没有本质的区别，可是别忘记，这是两张不同的网卡，你用ifconfig看一下</p>
<p><img src="https://s2.loli.net/2022/03/30/uaxs4mFhE8YSlbW.png" alt="image-20220330170822427"></p>
<p>明白了吧，网卡不同，它所在的端口也就不同。</p>
<h2 id="导入数据"><a href="#导入数据" class="headerlink" title="导入数据"></a>导入数据</h2><p>在hive中创建一个数据库，用来存放你系统的数据</p>
<pre class="line-numbers language-mysql" data-language="mysql"><code class="language-mysql">hive&gt; create database ecommerce;
OK
Time taken: 0.099 seconds
hive&gt; show databases;
OK
default
ecommerce
Time taken: 0.031 seconds, Fetched: 2 row(s)<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>在MySQL中建数据库，存放原始数据，这里的数据是假设由真实数据导入的，作为业务最初始的数据集。</p>
<pre class="line-numbers language-mysql" data-language="mysql"><code class="language-mysql">mysql&gt; create database ecommerce charset&#x3D;utf8;
Query OK, 1 row affected (0.01 sec)<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>

<h3 id="表设计"><a href="#表设计" class="headerlink" title="表设计"></a>表设计</h3><p>这些数据内容，主要可以分为：用户信息，商品信息，订单信息，运营相关信息，服务相关信息五大类，我们后续可以根据这个标准做标签建模。</p>
<ul>
<li>用户信息表（t_member）</li>
<li>用户地址表（t_member_addr）</li>
<li>商品信息表（t_commodity）</li>
<li>商品类别信息表（t_commodity_cate）</li>
<li>订单表（t_order）</li>
<li>订单商品表（t_order_commodity）</li>
<li>优惠券表（t_coupon）</li>
<li>用户优惠券表（t_coupon_member）</li>
<li>订单优惠券表（t_coupon_order）</li>
<li>快递表（t_delivery）</li>
<li>反馈表（t_feedback）</li>
<li>商店表（t_shop）</li>
<li>商家订单表（t_shop_order，订单对于卖家也有记录）</li>
<li>后台用户表（t_user，员工表）</li>
</ul>
<p>要打开mysql远程连接</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hengxing@hengxing:&#x2F;etc&#x2F;mysql&#x2F;mysql.conf.d$ vim mysqld.cnf<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>找到bind-address &#x3D; 127.0.0.1这一行<br>        改为bind-address &#x3D; 0.0.0.0即可</p>
<p>把ddl文件拖进navicat执行</p>
<img src="https://s2.loli.net/2022/03/30/o2aDupv3lzPebNS.png" alt="image-20220330184253874" style="zoom: 67%;" />

<img src="https://s2.loli.net/2022/03/30/AdHDMq1RiNSP9sJ.png" alt="image-20220330184331847" style="zoom:67%;" />

<p>一共14张表</p>
<h3 id="配置sqoop做数据迁移"><a href="#配置sqoop做数据迁移" class="headerlink" title="配置sqoop做数据迁移"></a>配置sqoop做数据迁移</h3><p>解压sqoop安装包</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hengxing@hengxing:~&#x2F;pkg$ sudo tar -zxvf sqoop-1.4.7.bin__hadoop-2.6.0.tar.gz -C &#x2F;usr&#x2F;local<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>为sqoop添加软连接（可选）</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hengxing@hengxing:&#x2F;usr&#x2F;local$ sudo ln -snf sqoop-1.4.7.bin__hadoop-2.6.0&#x2F; sqoop<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>添加mysql的连接器jar包</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hengxing@hengxing:~&#x2F;pkg&#x2F;mysql-connector-java-5.1.49$ cp mysql-connector-java-5.1.49-bin.jar &#x2F;usr&#x2F;local&#x2F;sqoop-1.4.7.bin__hadoop-2.6.0&#x2F;lib&#x2F;<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>另外注意，还应该把hive-site.xml拷贝到sqoop&#x2F;conf下面</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hengxing@hengxing:&#x2F;usr&#x2F;local&#x2F;sqoop$ cp &#x2F;usr&#x2F;local&#x2F;hive&#x2F;conf&#x2F;hive-site.xml conf&#x2F;<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>添加环境变量，便于以后执行命令</p>
<pre class="line-numbers language-sh" data-language="sh"><code class="language-sh">#sqoop
export SQOOP_HOME&#x3D;&#x2F;usr&#x2F;local&#x2F;sqoop
export PATH&#x3D;$PATH:$SQOOP_HOME&#x2F;bin<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<p>修改配置文件，加入hive的环境变量：</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hengxing@hengxing:&#x2F;usr&#x2F;local&#x2F;sqoop&#x2F;conf$ mv sqoop-env-template.sh sqoop-env.sh<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>写入如下内容</p>
<pre class="line-numbers language-sh" data-language="sh"><code class="language-sh">export HIVE_HOME&#x3D;&#x2F;usr&#x2F;local&#x2F;hive<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>上传<code>datamigrate.sh</code>脚本，检查脚本是否有错误，执行，脚本内容在后</p>
<pre class="line-numbers language-none"><code class="language-none">sudo apt-get install dos2unix #下载dos转unix工具
hengxing@hengxing:~&#x2F;profile_tool$ dos2unix datamigrate.sh 
dos2unix: converting file datamigrate.sh to Unix format ...
<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<pre class="line-numbers language-sh" data-language="sh"><code class="language-sh">#!&#x2F;bin&#x2F;bash

sq()
&#123;
sqoop import \
--connect jdbc:mysql:&#x2F;&#x2F;localhost:3306&#x2F;ecommerce \
--username root \
--password 123456 \
--table $1 \
--num-mappers 1 \
--hive-import \
--fields-terminated-by &quot;\t&quot; \
--hive-overwrite \
--hive-database ecommerce \
--hive-table $1
&#125;

sq t_commodity
sq t_commodity_cate
sq t_coupon
sq t_coupon_member
sq t_coupon_order
sq t_delivery
sq t_feedback
sq t_member
sq t_member_addr
sq t_order
sq t_order_commodity
sq t_shop
sq t_shop_order
sq t_user<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<h4 id="yarn-杀死-application-命令"><a href="#yarn-杀死-application-命令" class="headerlink" title="yarn 杀死 application 命令"></a>yarn 杀死 application 命令</h4><p><img src="https://s2.loli.net/2022/03/30/rEjN8LHM2iwocUJ.png" alt="image-20220330211003135"></p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">yarn application -kill application_1648627463824_0001<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>即可杀死</p>
<h4 id="hive没有找到类HiveConf"><a href="#hive没有找到类HiveConf" class="headerlink" title="hive没有找到类HiveConf"></a>hive没有找到类HiveConf</h4><pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">22&#x2F;03&#x2F;31 14:30:23 ERROR hive.HiveConfig: Could not load org.apache.hadoop.hive.conf.HiveConf. Make sure HIVE_CONF_DIR is set correctly.
22&#x2F;03&#x2F;31 14:30:23 ERROR tool.ImportTool: Import failed: java.io.IOException: java.lang.ClassNotFoundException: org.apache.hadoop.hive.conf.HiveConf<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>

<p>我们先看hive命令中有没有这个类</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hengxing@hengxing:&#x2F;usr&#x2F;local&#x2F;hive&#x2F;lib$ jar tf hive-common-2.3.7.jar | grep HiveConf.class
org&#x2F;apache&#x2F;hadoop&#x2F;hive&#x2F;conf&#x2F;HiveConf.class<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>

<p>有，只是没找到。所以我设置了下面的环境变量，注意是在<code>/etc/profile</code>下</p>
<pre class="line-numbers language-sh" data-language="sh"><code class="language-sh">export HIVE_CONF_DIR&#x3D;$HIVE_HOME&#x2F;conf
export HADOOP_CLASSPATH&#x3D;$HADOOP_CLASSPATH:$HIVE_HOME&#x2F;lib&#x2F;*<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>

<p>但是并没有什么卵用，网上还推荐了一步，将hive 里面的lib下的<code>hive-exec-**.jar</code> 放到sqoop 的lib 下，最后成功了。</p>
<h2 id="ES安装"><a href="#ES安装" class="headerlink" title="ES安装"></a>ES安装</h2><h3 id="下载、安装、配置"><a href="#下载、安装、配置" class="headerlink" title="下载、安装、配置"></a>下载、安装、配置</h3><p>地址:<a target="_blank" rel="noopener" href="https://www.elastic.co/cn/downloads/past-releases/elasticsearch-6-8-5">https://www.elastic.co/cn/downloads/past-releases/elasticsearch-6-8-5</a></p>
<p>下载后，上传并解压</p>
<p>把es放到&#x2F;usr&#x2F;local下，并创建软连接</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">sudo ln -snf elasticsearch-6.8.5&#x2F; es<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>修改配置文件</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hengxing@hengxing:&#x2F;usr&#x2F;local&#x2F;es$ cd config&#x2F;
hengxing@hengxing:&#x2F;usr&#x2F;local&#x2F;es&#x2F;config$ ls
elasticsearch.yml  log4j2.properties  roles.yml  users_roles
jvm.options        role_mapping.yml   users<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span></span></code></pre>

<p>修改elasticsearch.yml</p>
<pre class="line-numbers language-sh" data-language="sh"><code class="language-sh">#-----集群名称------
cluster.name: hengxing
# ------- Node名称 -----------
# Use a descriptive name for the node:
node.name: hengxing_node1
# ------------- Paths ---------------
# Path to directory where to store the data (separate multiple locations by comma):
path.data: &#x2F;home&#x2F;hengxing&#x2F;data&#x2F;esdata&#x2F;data
# Path to log files:
path.logs: &#x2F;home&#x2F;hengxing&#x2F;data&#x2F;esdata&#x2F;logs
# ------------------ Network -------------------
# Set the bind address to a specific IP (IPv4 or IPv6):0.0.0.0是允许所有IP访问
network.host: 0.0.0.0
# Set a custom port for HTTP:
http.port: 9200<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>修改jvm.options</p>
<pre class="line-numbers language-sh" data-language="sh"><code class="language-sh">## You should always set the min and max JVM heap
## size to the same value. For example, to set
## the heap to 4 GB, set:
##
-Xms1g
-Xmx1g<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<h3 id="启动命令"><a href="#启动命令" class="headerlink" title="启动命令"></a>启动命令</h3><pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">&#x2F;usr&#x2F;local&#x2F;es&#x2F;bin&#x2F;elasticsearch &amp;<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>访问地址<a target="_blank" rel="noopener" href="http://hengxing:9200/%EF%BC%8C%E5%8F%AF%E4%BB%A5%E7%9C%8B%E5%88%B0">http://hengxing:9200/，可以看到</a></p>
<pre class="line-numbers language-json" data-language="json"><code class="language-json"><span class="token punctuation">&#123;</span>
  <span class="token property">"name"</span> <span class="token operator">:</span> <span class="token string">"hengxing_node1"</span><span class="token punctuation">,</span>
  <span class="token property">"cluster_name"</span> <span class="token operator">:</span> <span class="token string">"hengxing"</span><span class="token punctuation">,</span>
  <span class="token property">"cluster_uuid"</span> <span class="token operator">:</span> <span class="token string">"NnCvOW_LRLmTKrdApgxWRw"</span><span class="token punctuation">,</span>
  <span class="token property">"version"</span> <span class="token operator">:</span> <span class="token punctuation">&#123;</span>
    <span class="token property">"number"</span> <span class="token operator">:</span> <span class="token string">"6.8.5"</span><span class="token punctuation">,</span>
    <span class="token property">"build_flavor"</span> <span class="token operator">:</span> <span class="token string">"default"</span><span class="token punctuation">,</span>
    <span class="token property">"build_type"</span> <span class="token operator">:</span> <span class="token string">"tar"</span><span class="token punctuation">,</span>
    <span class="token property">"build_hash"</span> <span class="token operator">:</span> <span class="token string">"78990e9"</span><span class="token punctuation">,</span>
    <span class="token property">"build_date"</span> <span class="token operator">:</span> <span class="token string">"2019-11-13T20:04:24.100411Z"</span><span class="token punctuation">,</span>
    <span class="token property">"build_snapshot"</span> <span class="token operator">:</span> <span class="token boolean">false</span><span class="token punctuation">,</span>
    <span class="token property">"lucene_version"</span> <span class="token operator">:</span> <span class="token string">"7.7.2"</span><span class="token punctuation">,</span>
    <span class="token property">"minimum_wire_compatibility_version"</span> <span class="token operator">:</span> <span class="token string">"5.6.0"</span><span class="token punctuation">,</span>
    <span class="token property">"minimum_index_compatibility_version"</span> <span class="token operator">:</span> <span class="token string">"5.0.0"</span>
  <span class="token punctuation">&#125;</span><span class="token punctuation">,</span>
  <span class="token property">"tagline"</span> <span class="token operator">:</span> <span class="token string">"You Know, for Search"</span>
<span class="token punctuation">&#125;</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<h3 id="报错"><a href="#报错" class="headerlink" title="报错"></a>报错</h3><p>[报错]max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [655360]</p>
<p>sudo vim &#x2F;etc&#x2F;sysctl.conf</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">vm.max_map_count&#x3D;655360<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>再执行<code>sysctl -p</code>即可</p>
<p>启动成功后可以访问</p>
<pre class="line-numbers language-json" data-language="json"><code class="language-json"><span class="token punctuation">&#123;</span>
  <span class="token property">"name"</span> <span class="token operator">:</span> <span class="token string">"hengxing_node1"</span><span class="token punctuation">,</span>
  <span class="token property">"cluster_name"</span> <span class="token operator">:</span> <span class="token string">"hengxing"</span><span class="token punctuation">,</span>
  <span class="token property">"cluster_uuid"</span> <span class="token operator">:</span> <span class="token string">"NnCvOW_LRLmTKrdApgxWRw"</span><span class="token punctuation">,</span>
  <span class="token property">"version"</span> <span class="token operator">:</span> <span class="token punctuation">&#123;</span>
    <span class="token property">"number"</span> <span class="token operator">:</span> <span class="token string">"6.8.5"</span><span class="token punctuation">,</span>
    <span class="token property">"build_flavor"</span> <span class="token operator">:</span> <span class="token string">"default"</span><span class="token punctuation">,</span>
    <span class="token property">"build_type"</span> <span class="token operator">:</span> <span class="token string">"tar"</span><span class="token punctuation">,</span>
    <span class="token property">"build_hash"</span> <span class="token operator">:</span> <span class="token string">"78990e9"</span><span class="token punctuation">,</span>
    <span class="token property">"build_date"</span> <span class="token operator">:</span> <span class="token string">"2019-11-13T20:04:24.100411Z"</span><span class="token punctuation">,</span>
    <span class="token property">"build_snapshot"</span> <span class="token operator">:</span> <span class="token boolean">false</span><span class="token punctuation">,</span>
    <span class="token property">"lucene_version"</span> <span class="token operator">:</span> <span class="token string">"7.7.2"</span><span class="token punctuation">,</span>
    <span class="token property">"minimum_wire_compatibility_version"</span> <span class="token operator">:</span> <span class="token string">"5.6.0"</span><span class="token punctuation">,</span>
    <span class="token property">"minimum_index_compatibility_version"</span> <span class="token operator">:</span> <span class="token string">"5.0.0"</span>
  <span class="token punctuation">&#125;</span><span class="token punctuation">,</span>
  <span class="token property">"tagline"</span> <span class="token operator">:</span> <span class="token string">"You Know, for Search"</span>
<span class="token punctuation">&#125;</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<h2 id="kibana安装"><a href="#kibana安装" class="headerlink" title="kibana安装"></a>kibana安装</h2><h3 id="下载、安装、配置-1"><a href="#下载、安装、配置-1" class="headerlink" title="下载、安装、配置"></a>下载、安装、配置</h3><p>【下载地址】<a target="_blank" rel="noopener" href="https://www.elastic.co/cn/downloads/past-releases/kibana-6-8-5">https://www.elastic.co/cn/downloads/past-releases/kibana-6-8-5</a></p>
<p>老一套，接着去改配置</p>
<pre class="line-numbers language-sh" data-language="sh"><code class="language-sh">server.host: &quot;hengxing&quot;<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>启动</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">&#x2F;usr&#x2F;local&#x2F;kibana&#x2F;bin&#x2F;kibana &amp;<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>报错</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">cause: undefined, path: [ &#39;basePath&#39; ] &#125; &#125;

 FATAL  Error: [server.basePath]: must start with a slash, don&#39;t end with one<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<p>这是因为写了basepath，并且报错写的很明显了。在这里可以注释掉</p>
<h3 id="如何使用"><a href="#如何使用" class="headerlink" title="如何使用"></a>如何使用</h3><p>在DevTools中 <code>PUT /usertag</code></p>
<p>再去Dashboard中的Index Management中就可以看到这个查询</p>
<h4 id="实例"><a href="#实例" class="headerlink" title="实例"></a>实例</h4><p>为了试验，我们添加一个查询<code>PUT /user</code>，然后添加一条数据</p>
<pre class="line-numbers language-json" data-language="json"><code class="language-json">POST /user/_doc/<span class="token number">1</span>
<span class="token punctuation">&#123;</span>
  <span class="token property">"name"</span><span class="token operator">:</span><span class="token string">"Tifa"</span><span class="token punctuation">,</span>
  <span class="token property">"age"</span><span class="token operator">:</span><span class="token number">17</span>
<span class="token punctuation">&#125;</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>能在控制台得到反馈</p>
<pre class="line-numbers language-json" data-language="json"><code class="language-json"><span class="token punctuation">&#123;</span>
  <span class="token property">"_index"</span> <span class="token operator">:</span> <span class="token string">"user"</span><span class="token punctuation">,</span>
  <span class="token property">"_type"</span> <span class="token operator">:</span> <span class="token string">"_doc"</span><span class="token punctuation">,</span>
  <span class="token property">"_id"</span> <span class="token operator">:</span> <span class="token string">"1"</span><span class="token punctuation">,</span>
  <span class="token property">"_version"</span> <span class="token operator">:</span> <span class="token number">1</span><span class="token punctuation">,</span>
  <span class="token property">"result"</span> <span class="token operator">:</span> <span class="token string">"created"</span><span class="token punctuation">,</span>
  <span class="token property">"_shards"</span> <span class="token operator">:</span> <span class="token punctuation">&#123;</span>
    <span class="token property">"total"</span> <span class="token operator">:</span> <span class="token number">2</span><span class="token punctuation">,</span>
    <span class="token property">"successful"</span> <span class="token operator">:</span> <span class="token number">1</span><span class="token punctuation">,</span>
    <span class="token property">"failed"</span> <span class="token operator">:</span> <span class="token number">0</span>
  <span class="token punctuation">&#125;</span><span class="token punctuation">,</span>
  <span class="token property">"_seq_no"</span> <span class="token operator">:</span> <span class="token number">0</span><span class="token punctuation">,</span>
  <span class="token property">"_primary_term"</span> <span class="token operator">:</span> <span class="token number">1</span>
<span class="token punctuation">&#125;</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>对这用户数据进行查询<code>GET /user/_search</code>，可以得到用户数据</p>
<pre class="line-numbers language-json" data-language="json"><code class="language-json"><span class="token punctuation">&#123;</span>
  <span class="token property">"took"</span> <span class="token operator">:</span> <span class="token number">5</span><span class="token punctuation">,</span>
  <span class="token property">"timed_out"</span> <span class="token operator">:</span> <span class="token boolean">false</span><span class="token punctuation">,</span>
  <span class="token property">"_shards"</span> <span class="token operator">:</span> <span class="token punctuation">&#123;</span>
    <span class="token property">"total"</span> <span class="token operator">:</span> <span class="token number">5</span><span class="token punctuation">,</span>
    <span class="token property">"successful"</span> <span class="token operator">:</span> <span class="token number">5</span><span class="token punctuation">,</span>
    <span class="token property">"skipped"</span> <span class="token operator">:</span> <span class="token number">0</span><span class="token punctuation">,</span>
    <span class="token property">"failed"</span> <span class="token operator">:</span> <span class="token number">0</span>
  <span class="token punctuation">&#125;</span><span class="token punctuation">,</span>
  <span class="token property">"hits"</span> <span class="token operator">:</span> <span class="token punctuation">&#123;</span>
    <span class="token property">"total"</span> <span class="token operator">:</span> <span class="token number">1</span><span class="token punctuation">,</span>
    <span class="token property">"max_score"</span> <span class="token operator">:</span> <span class="token number">1.0</span><span class="token punctuation">,</span>
    <span class="token property">"hits"</span> <span class="token operator">:</span> <span class="token punctuation">[</span>
      <span class="token punctuation">&#123;</span>
        <span class="token property">"_index"</span> <span class="token operator">:</span> <span class="token string">"user"</span><span class="token punctuation">,</span>
        <span class="token property">"_type"</span> <span class="token operator">:</span> <span class="token string">"_doc"</span><span class="token punctuation">,</span>
        <span class="token property">"_id"</span> <span class="token operator">:</span> <span class="token string">"1"</span><span class="token punctuation">,</span>
        <span class="token property">"_score"</span> <span class="token operator">:</span> <span class="token number">1.0</span><span class="token punctuation">,</span>
        <span class="token property">"_source"</span> <span class="token operator">:</span> <span class="token punctuation">&#123;</span>
          <span class="token property">"name"</span> <span class="token operator">:</span> <span class="token string">"Tifa"</span><span class="token punctuation">,</span>
          <span class="token property">"age"</span> <span class="token operator">:</span> <span class="token number">17</span>
        <span class="token punctuation">&#125;</span>
      <span class="token punctuation">&#125;</span>
    <span class="token punctuation">]</span>
  <span class="token punctuation">&#125;</span>
<span class="token punctuation">&#125;</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>若要进行条件查询，附加条件即可，不过我们还要多加两条数据，还是按照刚才的post方法</p>
<pre class="line-numbers language-json" data-language="json"><code class="language-json"><span class="token punctuation">&#123;</span>
      <span class="token property">"name"</span> <span class="token operator">:</span> <span class="token string">"Cloud"</span><span class="token punctuation">,</span>
      <span class="token property">"age"</span> <span class="token operator">:</span> <span class="token number">20</span>
<span class="token punctuation">&#125;</span><span class="token punctuation">,</span>
<span class="token punctuation">&#123;</span>
      <span class="token property">"name"</span> <span class="token operator">:</span> <span class="token string">"Tifa"</span><span class="token punctuation">,</span>
      <span class="token property">"age"</span> <span class="token operator">:</span> <span class="token number">17</span>
<span class="token punctuation">&#125;</span><span class="token punctuation">,</span>
<span class="token punctuation">&#123;</span>
      <span class="token property">"name"</span> <span class="token operator">:</span> <span class="token string">"Aurith"</span><span class="token punctuation">,</span>
      <span class="token property">"age"</span> <span class="token operator">:</span> <span class="token number">18</span>
<span class="token punctuation">&#125;</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<ol>
<li><p>查询名字为Cloud<strong>或</strong>年龄大于18小于20岁的</p>
<pre class="line-numbers language-json" data-language="json"><code class="language-json">GET /user/_search
<span class="token punctuation">&#123;</span>
  <span class="token property">"query"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span><span class="token property">"bool"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span>
    <span class="token property">"should"</span><span class="token operator">:</span> <span class="token punctuation">[</span>
      <span class="token punctuation">&#123;</span>
        <span class="token property">"match"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span>
          <span class="token property">"name"</span><span class="token operator">:</span> <span class="token string">"Cloud"</span>
        <span class="token punctuation">&#125;</span>
      <span class="token punctuation">&#125;</span><span class="token punctuation">,</span>
      <span class="token punctuation">&#123;</span>
        <span class="token property">"range"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span>
          <span class="token property">"age"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span>
            <span class="token property">"gte"</span><span class="token operator">:</span> <span class="token number">18</span><span class="token punctuation">,</span>
            <span class="token property">"lte"</span><span class="token operator">:</span> <span class="token number">20</span>
          <span class="token punctuation">&#125;</span>
        <span class="token punctuation">&#125;</span>
      <span class="token punctuation">&#125;</span>
    <span class="token punctuation">]</span>
  <span class="token punctuation">&#125;</span><span class="token punctuation">&#125;</span>
<span class="token punctuation">&#125;</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>结果：</p>
<pre class="line-numbers language-json" data-language="json"><code class="language-json"><span class="token punctuation">&#123;</span>
  <span class="token property">"took"</span> <span class="token operator">:</span> <span class="token number">8</span><span class="token punctuation">,</span>
  <span class="token property">"timed_out"</span> <span class="token operator">:</span> <span class="token boolean">false</span><span class="token punctuation">,</span>
  <span class="token property">"_shards"</span> <span class="token operator">:</span> <span class="token punctuation">&#123;</span>
    <span class="token property">"total"</span> <span class="token operator">:</span> <span class="token number">5</span><span class="token punctuation">,</span>
    <span class="token property">"successful"</span> <span class="token operator">:</span> <span class="token number">5</span><span class="token punctuation">,</span>
    <span class="token property">"skipped"</span> <span class="token operator">:</span> <span class="token number">0</span><span class="token punctuation">,</span>
    <span class="token property">"failed"</span> <span class="token operator">:</span> <span class="token number">0</span>
  <span class="token punctuation">&#125;</span><span class="token punctuation">,</span>
  <span class="token property">"hits"</span> <span class="token operator">:</span> <span class="token punctuation">&#123;</span>
    <span class="token property">"total"</span> <span class="token operator">:</span> <span class="token number">2</span><span class="token punctuation">,</span>
    <span class="token property">"max_score"</span> <span class="token operator">:</span> <span class="token number">1.287682</span><span class="token punctuation">,</span>
    <span class="token property">"hits"</span> <span class="token operator">:</span> <span class="token punctuation">[</span>
      <span class="token punctuation">&#123;</span>
        <span class="token property">"_index"</span> <span class="token operator">:</span> <span class="token string">"user"</span><span class="token punctuation">,</span>
        <span class="token property">"_type"</span> <span class="token operator">:</span> <span class="token string">"_doc"</span><span class="token punctuation">,</span>
        <span class="token property">"_id"</span> <span class="token operator">:</span> <span class="token string">"2"</span><span class="token punctuation">,</span>
        <span class="token property">"_score"</span> <span class="token operator">:</span> <span class="token number">1.287682</span><span class="token punctuation">,</span>
        <span class="token property">"_source"</span> <span class="token operator">:</span> <span class="token punctuation">&#123;</span>
          <span class="token property">"name"</span> <span class="token operator">:</span> <span class="token string">"Cloud"</span><span class="token punctuation">,</span>
          <span class="token property">"age"</span> <span class="token operator">:</span> <span class="token number">20</span>
        <span class="token punctuation">&#125;</span>
      <span class="token punctuation">&#125;</span><span class="token punctuation">,</span>
      <span class="token punctuation">&#123;</span>
        <span class="token property">"_index"</span> <span class="token operator">:</span> <span class="token string">"user"</span><span class="token punctuation">,</span>
        <span class="token property">"_type"</span> <span class="token operator">:</span> <span class="token string">"_doc"</span><span class="token punctuation">,</span>
        <span class="token property">"_id"</span> <span class="token operator">:</span> <span class="token string">"3"</span><span class="token punctuation">,</span>
        <span class="token property">"_score"</span> <span class="token operator">:</span> <span class="token number">1.0</span><span class="token punctuation">,</span>
        <span class="token property">"_source"</span> <span class="token operator">:</span> <span class="token punctuation">&#123;</span>
          <span class="token property">"name"</span> <span class="token operator">:</span> <span class="token string">"Aurith"</span><span class="token punctuation">,</span>
          <span class="token property">"age"</span> <span class="token operator">:</span> <span class="token number">18</span>
        <span class="token punctuation">&#125;</span>
      <span class="token punctuation">&#125;</span>
    <span class="token punctuation">]</span>
  <span class="token punctuation">&#125;</span>
<span class="token punctuation">&#125;</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>


</li>
<li><p>查询名字为Cloud<strong>且</strong>年龄大于18小于20岁的</p>
<p>结果：</p>
<pre class="line-numbers language-json" data-language="json"><code class="language-json"><span class="token punctuation">&#123;</span>
  <span class="token property">"took"</span> <span class="token operator">:</span> <span class="token number">8</span><span class="token punctuation">,</span>
  <span class="token property">"timed_out"</span> <span class="token operator">:</span> <span class="token boolean">false</span><span class="token punctuation">,</span>
  <span class="token property">"_shards"</span> <span class="token operator">:</span> <span class="token punctuation">&#123;</span>
    <span class="token property">"total"</span> <span class="token operator">:</span> <span class="token number">5</span><span class="token punctuation">,</span>
    <span class="token property">"successful"</span> <span class="token operator">:</span> <span class="token number">5</span><span class="token punctuation">,</span>
    <span class="token property">"skipped"</span> <span class="token operator">:</span> <span class="token number">0</span><span class="token punctuation">,</span>
    <span class="token property">"failed"</span> <span class="token operator">:</span> <span class="token number">0</span>
  <span class="token punctuation">&#125;</span><span class="token punctuation">,</span>
  <span class="token property">"hits"</span> <span class="token operator">:</span> <span class="token punctuation">&#123;</span>
    <span class="token property">"total"</span> <span class="token operator">:</span> <span class="token number">1</span><span class="token punctuation">,</span>
    <span class="token property">"max_score"</span> <span class="token operator">:</span> <span class="token number">1.287682</span><span class="token punctuation">,</span>
    <span class="token property">"hits"</span> <span class="token operator">:</span> <span class="token punctuation">[</span>
      <span class="token punctuation">&#123;</span>
        <span class="token property">"_index"</span> <span class="token operator">:</span> <span class="token string">"user"</span><span class="token punctuation">,</span>
        <span class="token property">"_type"</span> <span class="token operator">:</span> <span class="token string">"_doc"</span><span class="token punctuation">,</span>
        <span class="token property">"_id"</span> <span class="token operator">:</span> <span class="token string">"2"</span><span class="token punctuation">,</span>
        <span class="token property">"_score"</span> <span class="token operator">:</span> <span class="token number">1.287682</span><span class="token punctuation">,</span>
        <span class="token property">"_source"</span> <span class="token operator">:</span> <span class="token punctuation">&#123;</span>
          <span class="token property">"name"</span> <span class="token operator">:</span> <span class="token string">"Cloud"</span><span class="token punctuation">,</span>
          <span class="token property">"age"</span> <span class="token operator">:</span> <span class="token number">20</span>
        <span class="token punctuation">&#125;</span>
      <span class="token punctuation">&#125;</span>
    <span class="token punctuation">]</span>
  <span class="token punctuation">&#125;</span>
<span class="token punctuation">&#125;</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre></li>
</ol>
<h1 id="项目架构和工具类"><a href="#项目架构和工具类" class="headerlink" title="项目架构和工具类"></a>项目架构和工具类</h1><h2 id="IDEA项目"><a href="#IDEA项目" class="headerlink" title="IDEA项目"></a>IDEA项目</h2><p>新建Maven项目，参数如下：</p>
<pre class="line-numbers language-xml" data-language="xml"><code class="language-xml">GroupId：com.hengxing
Artifactid：EcommerceUserProfile<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>

<p>依赖文件POM，及版本信息</p>
<pre class="line-numbers language-xml" data-language="xml"><code class="language-xml"><span class="token comment">&lt;!--定义版本信息--></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>properties</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>java.version</span><span class="token punctuation">></span></span>1.8<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>java.version</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>spark.version</span><span class="token punctuation">></span></span>2.1.0<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>spark.version</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>scala.version</span><span class="token punctuation">></span></span>2.11<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>scala.version</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>springboot.version</span><span class="token punctuation">></span></span>2.2.0.RELEASE<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>springboot.version</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>elasticsearch.version</span><span class="token punctuation">></span></span>6.8.5<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>elasticsearch.version</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>lombok.version</span><span class="token punctuation">></span></span>1.18.10<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>lombok.version</span><span class="token punctuation">></span></span>
	<span class="token comment">&lt;!--自动生成getter/setter等方法，为配合此依赖还需安装插件lombok--></span>   
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>fastjson.version</span><span class="token punctuation">></span></span>1.2.59<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>fastjson.version</span><span class="token punctuation">></span></span>
	<span class="token comment">&lt;!--json转换器--></span>  
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>properties</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependencies</span><span class="token punctuation">></span></span>
    <span class="token comment">&lt;!--spring boot 相关依赖--></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.springframework.boot<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>spring-boot-starter-web<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>$&#123;springboot.version&#125;<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.springframework.boot<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>spring-boot-starter-thymeleaf<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>$&#123;springboot.version&#125;<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.springframework.boot<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>spring-boot-starter-data-elasticsearch<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>$&#123;springboot.version&#125;<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.springframework.boot<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>spring-boot-devtools<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>scope</span><span class="token punctuation">></span></span>runtime<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>scope</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>optional</span><span class="token punctuation">></span></span>true<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>optional</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>$&#123;springboot.version&#125;<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>

    <span class="token comment">&lt;!--spark 相关依赖，需要core、sql和hive--></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.apache.spark<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>spark-core_$&#123;scala.version&#125;<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>$&#123;spark.version&#125;<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>

    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.apache.spark<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>spark-sql_$&#123;scala.version&#125;<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>$&#123;spark.version&#125;<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.apache.spark<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>spark-hive_$&#123;scala.version&#125;<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>$&#123;spark.version&#125;<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>

    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.projectlombok<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>lombok<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>optional</span><span class="token punctuation">></span></span>true<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>optional</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>$&#123;lombok.version&#125;<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>

    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>com.alibaba<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>fastjson<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>$&#123;fastjson.version&#125;<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>

    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.elasticsearch<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>elasticsearch-hadoop<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>$&#123;elasticsearch.version&#125;<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.elasticsearch<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>elasticsearch<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>$&#123;elasticsearch.version&#125;<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>
    <span class="token comment">&lt;!-- Java High Level REST Client --></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.elasticsearch.client<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>elasticsearch-rest-high-level-client<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>$&#123;elasticsearch.version&#125;<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependencies</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>profiles</span><span class="token punctuation">></span></span>
    <span class="token comment">&lt;!--开发环境--></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>profile</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>id</span><span class="token punctuation">></span></span>dev<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>id</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>properties</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>build.profile.id</span><span class="token punctuation">></span></span>dev<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>build.profile.id</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>properties</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>activation</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>activeByDefault</span><span class="token punctuation">></span></span>true<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>activeByDefault</span><span class="token punctuation">></span></span>
            <span class="token comment">&lt;!-- 此标签可以切换生产环境和开发环境 --></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>activation</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>profile</span><span class="token punctuation">></span></span>
    <span class="token comment">&lt;!--生产环境--></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>profile</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>id</span><span class="token punctuation">></span></span>prod<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>id</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>properties</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>build.profile.id</span><span class="token punctuation">></span></span>prod<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>build.profile.id</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>properties</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>profile</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>profiles</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>build</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>plugins</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>plugin</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.springframework.boot<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>spring-boot-maven-plugin<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>plugin</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>plugin</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.apache.maven.plugins<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>maven-compiler-plugin<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>3.1<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>configuration</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>source</span><span class="token punctuation">></span></span>1.8<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>source</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>target</span><span class="token punctuation">></span></span>1.8<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>target</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>configuration</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>plugin</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>plugins</span><span class="token punctuation">></span></span>


    <span class="token comment">&lt;!--用resource定义打包时的资源文件--></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>resources</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>resource</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>directory</span><span class="token punctuation">></span></span>src/main/resources<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>directory</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>filtering</span><span class="token punctuation">></span></span>true<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>filtering</span><span class="token punctuation">></span></span>
            <span class="token comment">&lt;!-- 过滤其中的配置，符合条件的使用maven配置将其覆盖 --></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>excludes</span><span class="token punctuation">></span></span>
                <span class="token comment">&lt;!--除了这里声明的配置，其他的都可以被maven默认的配置覆盖--></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>exclude</span><span class="token punctuation">></span></span>application-*.properties<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>exclude</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>excludes</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>resource</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>resource</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>directory</span><span class="token punctuation">></span></span>src/main/resources<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>directory</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>filtering</span><span class="token punctuation">></span></span>false<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>filtering</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>includes</span><span class="token punctuation">></span></span><span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>include</span><span class="token punctuation">></span></span>application-$&#123;build.profile.id&#125;.properties<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>include</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>includes</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>resource</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>resources</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>build</span><span class="token punctuation">></span></span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>安装lombok插件</p>
<p><img src="https://s2.loli.net/2022/03/31/kbnKUB7RZyXJVrE.png" alt="image-20220331210805710"></p>
<p>设置远程连接，将服务器中hive的<code>hive-site.xml</code>文件拷贝至resources文件夹下</p>
<h2 id="编写类文件"><a href="#编写类文件" class="headerlink" title="编写类文件"></a>编写类文件</h2><h3 id="个人标签展示"><a href="#个人标签展示" class="headerlink" title="个人标签展示"></a>个人标签展示</h3><p>编写用户标签提取的文件，分成两种，一种是平台用户标签，另一种是个人用户标签提取，分别写两个包</p>
<p>SparkUtil.java</p>
<pre class="line-numbers language-java" data-language="java"><code class="language-java"><span class="token keyword">package</span> <span class="token namespace">com<span class="token punctuation">.</span>hengxing<span class="token punctuation">.</span>ecommerceuserprofile<span class="token punctuation">.</span>utils</span><span class="token punctuation">;</span>

<span class="token keyword">import</span> <span class="token namespace">org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>sql<span class="token punctuation">.</span></span><span class="token class-name">SparkSession</span><span class="token punctuation">;</span>

<span class="token keyword">public</span> <span class="token keyword">class</span> <span class="token class-name">SparkUtils</span> <span class="token punctuation">&#123;</span>
    <span class="token comment">//定义一个Spark Session的会话池</span>
    <span class="token keyword">private</span>  <span class="token keyword">static</span>  <span class="token class-name">ThreadLocal</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">SparkSession</span><span class="token punctuation">></span></span> sessionPool <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">ThreadLocal</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token punctuation">></span></span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

    <span class="token comment">//初始化spark session的方法</span>
    <span class="token keyword">public</span> <span class="token keyword">static</span> <span class="token class-name">SparkSession</span> <span class="token function">initSession</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">&#123;</span>
        <span class="token comment">//先判断会话池中是否由session，如果有，直接用，没就创建</span>
        <span class="token keyword">if</span><span class="token punctuation">(</span> sessionPool<span class="token punctuation">.</span><span class="token function">get</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">!=</span> <span class="token keyword">null</span><span class="token punctuation">)</span><span class="token punctuation">&#123;</span>
            <span class="token keyword">return</span> sessionPool<span class="token punctuation">.</span><span class="token function">get</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token punctuation">&#125;</span>
        <span class="token class-name">System</span><span class="token punctuation">.</span><span class="token function">setProperty</span><span class="token punctuation">(</span><span class="token string">"hadoop.home.dir"</span><span class="token punctuation">,</span><span class="token string">"C:\\tmp\\hadoop"</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token class-name">SparkSession</span> session <span class="token operator">=</span> <span class="token class-name">SparkSession</span><span class="token punctuation">.</span><span class="token function">builder</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
                <span class="token punctuation">.</span><span class="token function">appName</span><span class="token punctuation">(</span><span class="token string">"UserProfile-etl"</span><span class="token punctuation">)</span>
                <span class="token punctuation">.</span><span class="token function">master</span><span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span>
                <span class="token punctuation">.</span><span class="token function">config</span><span class="token punctuation">(</span><span class="token string">"es.nodes"</span><span class="token punctuation">,</span><span class="token string">"hengxing"</span><span class="token punctuation">)</span>  <span class="token comment">//elasticsearch的节点</span>
                <span class="token punctuation">.</span><span class="token function">config</span><span class="token punctuation">(</span><span class="token string">"es.port"</span><span class="token punctuation">,</span><span class="token string">"9200"</span><span class="token punctuation">)</span>   <span class="token comment">//elasticsearch的端口</span>
                <span class="token punctuation">.</span><span class="token function">config</span><span class="token punctuation">(</span><span class="token string">"ex.index.auto.create"</span><span class="token punctuation">,</span><span class="token string">"false"</span><span class="token punctuation">)</span>
                <span class="token punctuation">.</span><span class="token function">enableHiveSupport</span><span class="token punctuation">(</span><span class="token punctuation">)</span>    <span class="token comment">//启用hive支持</span>
                <span class="token punctuation">.</span><span class="token function">getOrCreate</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        sessionPool<span class="token punctuation">.</span><span class="token function">set</span><span class="token punctuation">(</span>session<span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token keyword">return</span> session<span class="token punctuation">;</span>
    <span class="token punctuation">&#125;</span>
<span class="token punctuation">&#125;</span>
<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>memberEtl.java</p>
<pre class="line-numbers language-java" data-language="java"><code class="language-java"><span class="token keyword">package</span> <span class="token namespace">com<span class="token punctuation">.</span>hengxing<span class="token punctuation">.</span>ecommerceuserprofile<span class="token punctuation">.</span>etls<span class="token punctuation">.</span>platform</span><span class="token punctuation">;</span>

<span class="token keyword">import</span> <span class="token namespace">com<span class="token punctuation">.</span>alibaba<span class="token punctuation">.</span>fastjson<span class="token punctuation">.</span></span>JSON<span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">com<span class="token punctuation">.</span>hengxing<span class="token punctuation">.</span>ecommerceuserprofile<span class="token punctuation">.</span>utils<span class="token punctuation">.</span></span><span class="token class-name">SparkUtils</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">lombok<span class="token punctuation">.</span></span><span class="token class-name">Data</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>sql<span class="token punctuation">.</span></span><span class="token class-name">Dataset</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>sql<span class="token punctuation">.</span></span><span class="token class-name">Row</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>sql<span class="token punctuation">.</span></span><span class="token class-name">SparkSession</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">java<span class="token punctuation">.</span>util<span class="token punctuation">.</span></span><span class="token class-name">List</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">java<span class="token punctuation">.</span>util<span class="token punctuation">.</span>stream<span class="token punctuation">.</span></span><span class="token class-name">Collectors</span><span class="token punctuation">;</span>

<span class="token keyword">public</span> <span class="token keyword">class</span> <span class="token class-name">MemberEtl</span> <span class="token punctuation">&#123;</span>
    <span class="token keyword">public</span> <span class="token keyword">static</span> <span class="token keyword">void</span> <span class="token function">main</span><span class="token punctuation">(</span><span class="token class-name">String</span><span class="token punctuation">[</span><span class="token punctuation">]</span> args<span class="token punctuation">)</span> <span class="token punctuation">&#123;</span>
        <span class="token class-name">System</span><span class="token punctuation">.</span><span class="token function">setProperty</span><span class="token punctuation">(</span><span class="token string">"hadoop.home.dir"</span><span class="token punctuation">,</span><span class="token string">"C:\\tmp\\hadoop"</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token comment">//初始化一个sparksession</span>
        <span class="token class-name">SparkSession</span> session <span class="token operator">=</span> <span class="token class-name">SparkUtils</span><span class="token punctuation">.</span><span class="token function">initSession</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

<span class="token comment">//        写sql查询想要的数据</span>
        <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">MemberSex</span><span class="token punctuation">></span></span> memberSexes <span class="token operator">=</span> <span class="token function">memberSexEtl</span><span class="token punctuation">(</span>session<span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">MemberChannel</span><span class="token punctuation">></span></span> memberChannels <span class="token operator">=</span> <span class="token function">memberChannelEtl</span><span class="token punctuation">(</span>session<span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">MemberMpSub</span><span class="token punctuation">></span></span> memberMpSubs <span class="token operator">=</span> <span class="token function">memberMpSubEtl</span><span class="token punctuation">(</span>session<span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token class-name">MemberHeat</span> memberHeat <span class="token operator">=</span> <span class="token function">memberHeatEtl</span><span class="token punctuation">(</span>session<span class="token punctuation">)</span><span class="token punctuation">;</span>

<span class="token comment">//        拼成想要展示的结果，给前端</span>
        <span class="token class-name">MemberVo</span> memberVo <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">MemberVo</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

        memberVo<span class="token punctuation">.</span><span class="token function">setMemberSexes</span><span class="token punctuation">(</span>memberSexes<span class="token punctuation">)</span><span class="token punctuation">;</span>
        memberVo<span class="token punctuation">.</span><span class="token function">setMemberChannels</span><span class="token punctuation">(</span>memberChannels<span class="token punctuation">)</span><span class="token punctuation">;</span>
        memberVo<span class="token punctuation">.</span><span class="token function">setMemberMpSubs</span><span class="token punctuation">(</span>memberMpSubs<span class="token punctuation">)</span><span class="token punctuation">;</span>
        memberVo<span class="token punctuation">.</span><span class="token function">setMemberHeat</span><span class="token punctuation">(</span>memberHeat<span class="token punctuation">)</span><span class="token punctuation">;</span>

        <span class="token comment">//控制台打印</span>
        <span class="token class-name">System</span><span class="token punctuation">.</span>out<span class="token punctuation">.</span><span class="token function">println</span><span class="token punctuation">(</span><span class="token string">"=====ETL数据来啦=====\n"</span> <span class="token operator">+</span> JSON<span class="token punctuation">.</span><span class="token function">toJSONString</span><span class="token punctuation">(</span>memberVo<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

    <span class="token punctuation">&#125;</span>

    <span class="token comment">//统计平台用户性别分布</span>
    <span class="token keyword">private</span> <span class="token keyword">static</span> <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">MemberSex</span><span class="token punctuation">></span></span> <span class="token function">memberSexEtl</span><span class="token punctuation">(</span><span class="token class-name">SparkSession</span> session<span class="token punctuation">)</span> <span class="token punctuation">&#123;</span>
        <span class="token comment">//写sql，查询得到一个dataset</span>
        <span class="token class-name">Dataset</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Row</span><span class="token punctuation">></span></span> dataset <span class="token operator">=</span> session<span class="token punctuation">.</span><span class="token function">sql</span><span class="token punctuation">(</span><span class="token string">"select sex as memberSex, count(id) as sexCount "</span> <span class="token operator">+</span>
                <span class="token string">"from ecommerce.t_member group by sex"</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token comment">//先将dataset转成list</span>
        <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">String</span><span class="token punctuation">></span></span> list <span class="token operator">=</span> dataset<span class="token punctuation">.</span><span class="token function">toJSON</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">collectAsList</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token comment">//将list转成流，进行每一行数据的遍历，转成memberSex</span>
        <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">MemberSex</span><span class="token punctuation">></span></span> result <span class="token operator">=</span> list<span class="token punctuation">.</span><span class="token function">stream</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
                <span class="token punctuation">.</span><span class="token function">map</span><span class="token punctuation">(</span>str <span class="token operator">-></span> JSON<span class="token punctuation">.</span><span class="token function">parseObject</span><span class="token punctuation">(</span>str<span class="token punctuation">,</span> <span class="token class-name">MemberSex</span><span class="token punctuation">.</span><span class="token keyword">class</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
                <span class="token punctuation">.</span><span class="token function">collect</span><span class="token punctuation">(</span><span class="token class-name">Collectors</span><span class="token punctuation">.</span><span class="token function">toList</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token keyword">return</span> result<span class="token punctuation">;</span>
    <span class="token punctuation">&#125;</span>

    <span class="token comment">//用户注册渠道的分布统计</span>
    <span class="token keyword">public</span> <span class="token keyword">static</span> <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">MemberChannel</span><span class="token punctuation">></span></span> <span class="token function">memberChannelEtl</span><span class="token punctuation">(</span><span class="token class-name">SparkSession</span> session<span class="token punctuation">)</span> <span class="token punctuation">&#123;</span>
        <span class="token class-name">Dataset</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Row</span><span class="token punctuation">></span></span> dataset <span class="token operator">=</span> session<span class="token punctuation">.</span><span class="token function">sql</span><span class="token punctuation">(</span>
                <span class="token string">"select member_channel as memberChannel, count(id) as channelCount "</span> <span class="token operator">+</span>
                        <span class="token string">" from ecommerce.t_member group by member_channel"</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

        <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">String</span><span class="token punctuation">></span></span> list <span class="token operator">=</span> dataset<span class="token punctuation">.</span><span class="token function">toJSON</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">collectAsList</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

        <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">MemberChannel</span><span class="token punctuation">></span></span> result <span class="token operator">=</span> list<span class="token punctuation">.</span><span class="token function">stream</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
                <span class="token punctuation">.</span><span class="token function">map</span><span class="token punctuation">(</span>str <span class="token operator">-></span> JSON<span class="token punctuation">.</span><span class="token function">parseObject</span><span class="token punctuation">(</span>str<span class="token punctuation">,</span> <span class="token class-name">MemberChannel</span><span class="token punctuation">.</span><span class="token keyword">class</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
                <span class="token punctuation">.</span><span class="token function">collect</span><span class="token punctuation">(</span><span class="token class-name">Collectors</span><span class="token punctuation">.</span><span class="token function">toList</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

        <span class="token keyword">return</span> result<span class="token punctuation">;</span>
    <span class="token punctuation">&#125;</span>



    <span class="token comment">/*
    * sql中if()的用法
    * if(&lt;筛选条件>,&lt;匹配时返回>,&lt;不匹配时返回>)
    * */</span>
    <span class="token comment">//媒体平台关注分布统计</span>
    <span class="token keyword">public</span> <span class="token keyword">static</span> <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">MemberMpSub</span><span class="token punctuation">></span></span> <span class="token function">memberMpSubEtl</span><span class="token punctuation">(</span><span class="token class-name">SparkSession</span> session<span class="token punctuation">)</span> <span class="token punctuation">&#123;</span>
        <span class="token class-name">Dataset</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Row</span><span class="token punctuation">></span></span> sub <span class="token operator">=</span> session<span class="token punctuation">.</span><span class="token function">sql</span><span class="token punctuation">(</span>
                <span class="token string">"select count(if(mp_open_id !='null',true,null)) as subCount, "</span> <span class="token operator">+</span>
                        <span class="token string">" count(if(mp_open_id ='null',true,null)) as unSubCount "</span> <span class="token operator">+</span>
                        <span class="token string">" from ecommerce.t_member"</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

        <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">String</span><span class="token punctuation">></span></span> list <span class="token operator">=</span> sub<span class="token punctuation">.</span><span class="token function">toJSON</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">collectAsList</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

        <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">MemberMpSub</span><span class="token punctuation">></span></span> result <span class="token operator">=</span> list<span class="token punctuation">.</span><span class="token function">stream</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
                <span class="token punctuation">.</span><span class="token function">map</span><span class="token punctuation">(</span>str <span class="token operator">-></span> JSON<span class="token punctuation">.</span><span class="token function">parseObject</span><span class="token punctuation">(</span>str<span class="token punctuation">,</span> <span class="token class-name">MemberMpSub</span><span class="token punctuation">.</span><span class="token keyword">class</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
                <span class="token punctuation">.</span><span class="token function">collect</span><span class="token punctuation">(</span><span class="token class-name">Collectors</span><span class="token punctuation">.</span><span class="token function">toList</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

        <span class="token keyword">return</span> result<span class="token punctuation">;</span>
    <span class="token punctuation">&#125;</span>


    <span class="token comment">//平台用户热度分布统计</span>
    <span class="token keyword">private</span> <span class="token keyword">static</span> <span class="token class-name">MemberHeat</span> <span class="token function">memberHeatEtl</span><span class="token punctuation">(</span><span class="token class-name">SparkSession</span> session<span class="token punctuation">)</span> <span class="token punctuation">&#123;</span>
        <span class="token comment">//reg，complete从用户表中提取</span>
        <span class="token class-name">Dataset</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Row</span><span class="token punctuation">></span></span> reg_complete_count <span class="token operator">=</span> session<span class="token punctuation">.</span><span class="token function">sql</span><span class="token punctuation">(</span>
                <span class="token string">"select count(if(phone = 'null',true,null)) as reg,"</span> <span class="token operator">+</span>
                <span class="token string">" count(if(phone != 'null',true,null)) as complete"</span> <span class="token operator">+</span>
                <span class="token string">" from ecommerce.t_member"</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token comment">//order, orderAgain从订单表中提取</span>
        <span class="token class-name">Dataset</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Row</span><span class="token punctuation">></span></span> order_andAgain_count <span class="token operator">=</span> session<span class="token punctuation">.</span><span class="token function">sql</span><span class="token punctuation">(</span><span class="token string">"select"</span> <span class="token operator">+</span>
                <span class="token string">" count(if(t.orderCount = 1,true,null)) as order,"</span> <span class="token operator">+</span>
                <span class="token string">" count(if(t.orderCount >= 2,true,null)) as orderAgain"</span> <span class="token operator">+</span>
                <span class="token string">" from (select count(order_id) as orderCount, member_id from ecommerce.t_order group by member_id) as t"</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token comment">//coupon,买了消费券人数，从t_coupon_member表提取，对member_id进行去重，然后count</span>
        <span class="token class-name">Dataset</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Row</span><span class="token punctuation">></span></span> coupon_count <span class="token operator">=</span> session<span class="token punctuation">.</span><span class="token function">sql</span><span class="token punctuation">(</span><span class="token string">"select"</span> <span class="token operator">+</span>
                <span class="token string">" count(distinct member_id) as coupon from ecommerce.t_coupon_member "</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token comment">//最终合并</span>
        <span class="token class-name">Dataset</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Row</span><span class="token punctuation">></span></span> heat <span class="token operator">=</span> coupon_count<span class="token punctuation">.</span><span class="token function">crossJoin</span><span class="token punctuation">(</span>reg_complete_count<span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">crossJoin</span><span class="token punctuation">(</span>order_andAgain_count<span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token comment">//先将dataset转成list</span>
        <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">String</span><span class="token punctuation">></span></span> list <span class="token operator">=</span> heat<span class="token punctuation">.</span><span class="token function">toJSON</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">collectAsList</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token comment">//这一次我们只有一条数据，取出第一个就可以了，这里是为了复用代码</span>
        <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">MemberHeat</span><span class="token punctuation">></span></span> result <span class="token operator">=</span> list<span class="token punctuation">.</span><span class="token function">stream</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
                <span class="token punctuation">.</span><span class="token function">map</span><span class="token punctuation">(</span>str <span class="token operator">-></span> JSON<span class="token punctuation">.</span><span class="token function">parseObject</span><span class="token punctuation">(</span>str<span class="token punctuation">,</span> <span class="token class-name">MemberHeat</span><span class="token punctuation">.</span><span class="token keyword">class</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
                <span class="token punctuation">.</span><span class="token function">collect</span><span class="token punctuation">(</span><span class="token class-name">Collectors</span><span class="token punctuation">.</span><span class="token function">toList</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token keyword">return</span> result<span class="token punctuation">.</span><span class="token function">get</span><span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
    <span class="token punctuation">&#125;</span>

    <span class="token comment">//    定义一个最终想要生成的ViewObject，用来展示饼图</span>
    <span class="token annotation punctuation">@Data</span>
    <span class="token keyword">static</span> <span class="token keyword">class</span> <span class="token class-name">MemberVo</span><span class="token punctuation">&#123;</span>
        <span class="token comment">//由四部分构成</span>
        <span class="token keyword">private</span> <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">MemberSex</span><span class="token punctuation">></span></span> memberSexes<span class="token punctuation">;</span>    <span class="token comment">//性别统计信息</span>
        <span class="token keyword">private</span> <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">MemberChannel</span><span class="token punctuation">></span></span> memberChannels<span class="token punctuation">;</span> <span class="token comment">//渠道统计信息</span>
        <span class="token keyword">private</span> <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">MemberMpSub</span><span class="token punctuation">></span></span> memberMpSubs<span class="token punctuation">;</span> <span class="token comment">//是否关注媒体平台统计</span>
        <span class="token keyword">private</span> <span class="token class-name">MemberHeat</span> memberHeat<span class="token punctuation">;</span>  <span class="token comment">//用户热度统计</span>
    <span class="token punctuation">&#125;</span>

    <span class="token annotation punctuation">@Data</span>
    <span class="token keyword">static</span> <span class="token keyword">class</span> <span class="token class-name">MemberSex</span><span class="token punctuation">&#123;</span>
        <span class="token keyword">private</span> <span class="token class-name">Integer</span> memberSex<span class="token punctuation">;</span><span class="token comment">//性别编号</span>
        <span class="token keyword">private</span> <span class="token class-name">Integer</span> sexCount<span class="token punctuation">;</span>   <span class="token comment">//当前性别的count数量</span>
    <span class="token punctuation">&#125;</span>

    <span class="token annotation punctuation">@Data</span>
    <span class="token keyword">static</span> <span class="token keyword">class</span> <span class="token class-name">MemberChannel</span><span class="token punctuation">&#123;</span>
        <span class="token keyword">private</span> <span class="token class-name">Integer</span> memberChannel<span class="token punctuation">;</span>      <span class="token comment">//渠道编号</span>
        <span class="token keyword">private</span> <span class="token class-name">Integer</span> channelCount<span class="token punctuation">;</span>
    <span class="token punctuation">&#125;</span>
    <span class="token annotation punctuation">@Data</span>
    <span class="token keyword">static</span> <span class="token keyword">class</span> <span class="token class-name">MemberMpSub</span><span class="token punctuation">&#123;</span>
        <span class="token keyword">private</span> <span class="token class-name">Integer</span> subCount<span class="token punctuation">;</span>   <span class="token comment">//已关注的用户统计数</span>
        <span class="token keyword">private</span> <span class="token class-name">Integer</span> unSubCount<span class="token punctuation">;</span>
    <span class="token punctuation">&#125;</span>
    <span class="token annotation punctuation">@Data</span>
    <span class="token keyword">static</span> <span class="token keyword">class</span> <span class="token class-name">MemberHeat</span><span class="token punctuation">&#123;</span>
        <span class="token keyword">private</span> <span class="token class-name">Integer</span> reg<span class="token punctuation">;</span>        <span class="token comment">//只注册未填手机号的用户</span>
        <span class="token keyword">private</span> <span class="token class-name">Integer</span> complete<span class="token punctuation">;</span>   <span class="token comment">//填了手机号的</span>
        <span class="token keyword">private</span> <span class="token class-name">Integer</span> order<span class="token punctuation">;</span>    <span class="token comment">//下过单</span>
        <span class="token keyword">private</span> <span class="token class-name">Integer</span> orderAgain<span class="token punctuation">;</span> <span class="token comment">//多次下单</span>
        <span class="token keyword">private</span> <span class="token class-name">Integer</span> coupon<span class="token punctuation">;</span>     <span class="token comment">//买过消费券的用户统计数</span>
    <span class="token punctuation">&#125;</span>
<span class="token punctuation">&#125;</span>
<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>正确配置环境并使用，需要以下几个条件：</p>
<blockquote>
<p>1、打开hive、spark、Hadoop以及hive元数据服务</p>
<p>2、在resource下放hive-site.xml</p>
<p>3、配置hive on spark</p>
<p>4、打开hive支持</p>
</blockquote>
<h4 id="报错-1"><a href="#报错-1" class="headerlink" title="报错"></a>报错</h4><h5 id="Caused-by-java-lang-NullPointerException"><a href="#Caused-by-java-lang-NullPointerException" class="headerlink" title="Caused by: java.lang.NullPointerException"></a>Caused by: java.lang.NullPointerException</h5><p>编写完文件后连接测试</p>
<p>报错</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">22:30:37.257 [main] WARN hive.ql.metadata.Hive - Failed to access metastore. This class should not accessed in runtime.
org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient
	at 
Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient
Caused by: java.lang.reflect.InvocationTargetException: null<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>这是完整的：</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">00:02:42.923 [main] WARN hive.ql.metadata.Hive - Failed to access metastore. This class should not accessed in runtime.
org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient
	at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1236)
	at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174)
	at org.apache.hadoop.hive.ql.metadata.Hive.&lt;clinit&gt;(Hive.java:166)
	at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503)
	at org.apache.spark.sql.hive.client.HiveClientImpl.&lt;init&gt;(HiveClientImpl.scala:192)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:264)
	at org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:366)
	at org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:270)
	at org.apache.spark.sql.hive.HiveExternalCatalog.&lt;init&gt;(HiveExternalCatalog.scala:65)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at org.apache.spark.sql.internal.SharedState$.org$apache$spark$sql$internal$SharedState$$reflect(SharedState.scala:166)
	at org.apache.spark.sql.internal.SharedState.&lt;init&gt;(SharedState.scala:86)
	at org.apache.spark.sql.SparkSession$$anonfun$sharedState$1.apply(SparkSession.scala:101)
	at org.apache.spark.sql.SparkSession$$anonfun$sharedState$1.apply(SparkSession.scala:101)
	at scala.Option.getOrElse(Option.scala:121)
	at org.apache.spark.sql.SparkSession.sharedState$lzycompute(SparkSession.scala:101)
	at org.apache.spark.sql.SparkSession.sharedState(SparkSession.scala:100)
	at org.apache.spark.sql.internal.SessionState.&lt;init&gt;(SessionState.scala:157)
	at org.apache.spark.sql.hive.HiveSessionState.&lt;init&gt;(HiveSessionState.scala:32)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at org.apache.spark.sql.SparkSession$.org$apache$spark$sql$SparkSession$$reflect(SparkSession.scala:978)
	at org.apache.spark.sql.SparkSession.sessionState$lzycompute(SparkSession.scala:110)
	at org.apache.spark.sql.SparkSession.sessionState(SparkSession.scala:109)
	at org.apache.spark.sql.SparkSession$Builder$$anonfun$getOrCreate$5.apply(SparkSession.scala:878)
	at org.apache.spark.sql.SparkSession$Builder$$anonfun$getOrCreate$5.apply(SparkSession.scala:878)
	at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:99)
	at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:99)
	at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:230)
	at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40)
	at scala.collection.mutable.HashMap.foreach(HashMap.scala:99)
	at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:878)
	at com.hengxing.ecommerceuserprofile.utils.SparkUtils.initSession(SparkUtils.java:32)
	at com.hengxing.ecommerceuserprofile.etls.platform.MemberEtl.main(MemberEtl.java:24)
Caused by: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient
	at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1523)
	at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.&lt;init&gt;(RetryingMetaStoreClient.java:86)
	at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132)
	at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104)
	at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005)
	at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024)
	at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234)
	... 42 common frames omitted
Caused by: java.lang.reflect.InvocationTargetException: null
	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521)
	... 48 common frames omitted
Caused by: java.lang.NullPointerException: null
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1012)
	at org.apache.hadoop.util.Shell.runCommand(Shell.java:404)
	at org.apache.hadoop.util.Shell.run(Shell.java:379)
	at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:589)
	at org.apache.hadoop.util.Shell.execCommand(Shell.java:678)
	at org.apache.hadoop.util.Shell.execCommand(Shell.java:661)
	at org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getUnixGroups(ShellBasedUnixGroupsMapping.java:83)
	at org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getGroups(ShellBasedUnixGroupsMapping.java:52)
	at org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback.getGroups(JniBasedUnixGroupsMappingWithFallback.java:50)
	at org.apache.hadoop.security.Groups.getGroups(Groups.java:89)
	at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1352)
	at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.open(HiveMetaStoreClient.java:436)
	at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.&lt;init&gt;(HiveMetaStoreClient.java:236)
	at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.&lt;init&gt;(SessionHiveMetaStoreClient.java:74)
	... 53 common frames omitted
00:02:42.924 [main] INFO hive.metastore - Trying to connect to metastore with URI thrift:&#x2F;&#x2F;hengxing:9083
Exception in thread &quot;main&quot; java.lang.IllegalArgumentException: Error while instantiating &#39;org.apache.spark.sql.hive.HiveSessionState&#39;:
	at org.apache.spark.sql.SparkSession$.org$apache$spark$sql$SparkSession$$reflect(SparkSession.scala:981)
	at org.apache.spark.sql.SparkSession.sessionState$lzycompute(SparkSession.scala:110)
	at org.apache.spark.sql.SparkSession.sessionState(SparkSession.scala:109)
	at org.apache.spark.sql.SparkSession$Builder$$anonfun$getOrCreate$5.apply(SparkSession.scala:878)
	at org.apache.spark.sql.SparkSession$Builder$$anonfun$getOrCreate$5.apply(SparkSession.scala:878)
	at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:99)
	at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:99)
	at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:230)
	at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40)
	at scala.collection.mutable.HashMap.foreach(HashMap.scala:99)
	at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:878)
	at com.hengxing.ecommerceuserprofile.utils.SparkUtils.initSession(SparkUtils.java:32)
	at com.hengxing.ecommerceuserprofile.etls.platform.MemberEtl.main(MemberEtl.java:24)
Caused by: java.lang.reflect.InvocationTargetException
	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at org.apache.spark.sql.SparkSession$.org$apache$spark$sql$SparkSession$$reflect(SparkSession.scala:978)
	... 12 more
Caused by: java.lang.IllegalArgumentException: Error while instantiating &#39;org.apache.spark.sql.hive.HiveExternalCatalog&#39;:
	at org.apache.spark.sql.internal.SharedState$.org$apache$spark$sql$internal$SharedState$$reflect(SharedState.scala:169)
	at org.apache.spark.sql.internal.SharedState.&lt;init&gt;(SharedState.scala:86)
	at org.apache.spark.sql.SparkSession$$anonfun$sharedState$1.apply(SparkSession.scala:101)
	at org.apache.spark.sql.SparkSession$$anonfun$sharedState$1.apply(SparkSession.scala:101)
	at scala.Option.getOrElse(Option.scala:121)
	at org.apache.spark.sql.SparkSession.sharedState$lzycompute(SparkSession.scala:101)
	at org.apache.spark.sql.SparkSession.sharedState(SparkSession.scala:100)
	at org.apache.spark.sql.internal.SessionState.&lt;init&gt;(SessionState.scala:157)
	at org.apache.spark.sql.hive.HiveSessionState.&lt;init&gt;(HiveSessionState.scala:32)
	... 17 more
Caused by: java.lang.reflect.InvocationTargetException
	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at org.apache.spark.sql.internal.SharedState$.org$apache$spark$sql$internal$SharedState$$reflect(SharedState.scala:166)
	... 25 more
Caused by: java.lang.reflect.InvocationTargetException
	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:264)
	at org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:366)
	at org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:270)
	at org.apache.spark.sql.hive.HiveExternalCatalog.&lt;init&gt;(HiveExternalCatalog.scala:65)
	... 30 more
Caused by: java.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient
	at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:522)
	at org.apache.spark.sql.hive.client.HiveClientImpl.&lt;init&gt;(HiveClientImpl.scala:192)
	... 38 more
Caused by: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient
	at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1523)
	at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.&lt;init&gt;(RetryingMetaStoreClient.java:86)
	at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132)
	at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104)
	at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005)
	at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024)
	at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503)
	... 39 more
Caused by: java.lang.reflect.InvocationTargetException
	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521)
	... 45 more
Caused by: java.lang.NullPointerException
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1012)
	at org.apache.hadoop.util.Shell.runCommand(Shell.java:404)
	at org.apache.hadoop.util.Shell.run(Shell.java:379)
	at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:589)
	at org.apache.hadoop.util.Shell.execCommand(Shell.java:678)
	at org.apache.hadoop.util.Shell.execCommand(Shell.java:661)
	at org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getUnixGroups(ShellBasedUnixGroupsMapping.java:83)
	at org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getGroups(ShellBasedUnixGroupsMapping.java:52)
	at org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback.getGroups(JniBasedUnixGroupsMappingWithFallback.java:50)
	at org.apache.hadoop.security.Groups.getGroups(Groups.java:89)
	at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1352)
	at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.open(HiveMetaStoreClient.java:436)
	at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.&lt;init&gt;(HiveMetaStoreClient.java:236)
	at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.&lt;init&gt;(SessionHiveMetaStoreClient.java:74)
	... 50 more<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>我以为是没开metastore服务，但是打开之后还是报错</p>
<p>测试一下服务有没有正常启动</p>
<p><img src="https://s2.loli.net/2022/04/01/KwWPz3bgVijA1dy.png" alt="image-20220401225351386"></p>
<p>说明启动起来了</p>
<p>又一想，sparkshell能不能启动呢，果然启动的时候报错，说连不上hive，并且有两个包引入错误</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">Caused by: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient
······
&lt;console&gt;:14: error: not found: value spark
       import spark.implicits._
              ^
&lt;console&gt;:14: error: not found: value spark
       import spark.sql
              ^<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>虽然不知道它是怎么想着连接hive的，但是总要试着解决，去网上查了一下sparkonhive的配置方法：</p>
<blockquote>
<p>配置spark on hive很简单，把hive-site.xml软链接到spark的conf目录下即可，spark会自动读取该配置来识别metastore的地址。</p>
</blockquote>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">ln -snf &#x2F;usr&#x2F;local&#x2F;hive&#x2F;conf&#x2F;hive-site.xml &#x2F;usr&#x2F;local&#x2F;spark&#x2F;conf&#x2F;hive-site.xml<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>再重新启动sparkshell，好了，现在没有上面那个报错了，并且成功查询到hive中的数据</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">Spark context Web UI available at http:&#x2F;&#x2F;192.168.59.140:4040
Spark context available as &#39;sc&#39; (master &#x3D; local[*], app id &#x3D; local-1648828695614).
Spark session available as &#39;spark&#39;.
Welcome to
      ____              __
     &#x2F; __&#x2F;__  ___ _____&#x2F; &#x2F;__
    _\ \&#x2F; _ \&#x2F; _ &#96;&#x2F; __&#x2F;  &#39;_&#x2F;
   &#x2F;___&#x2F; .__&#x2F;\_,_&#x2F;_&#x2F; &#x2F;_&#x2F;\_\   version 2.1.0
      &#x2F;_&#x2F;
         
Using Scala version 2.11.8 (OpenJDK 64-Bit Server VM, Java 1.8.0_292)
Type in expressions to have them evaluated.
Type :help for more information.

scala&gt; spark.sql(&quot;select * from ecommerce.t_user&quot;).show(false)
+---+---------+-------------+-----------+-------------------+-------------------+
|id |user_name|user_password|user_phone |create_time        |update_time        |
+---+---------+-------------+-----------+-------------------+-------------------+
|1  |一号员工     |111111       |15678990989|2019-08-01 08:31:46|2019-08-01 08:31:46|
|2  |二号员工     |111111       |15678990989|2019-08-01 08:31:46|2019-08-01 08:31:46|
|3  |三号员工     |111111       |15678990989|2019-08-01 08:31:46|2019-08-01 08:31:46|
+---+---------+-------------+-----------+-------------------+-------------------+
<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>但这只是本地的sparkshell没问题，程序还是有报错</p>
<p>咨询了一位学长，学长给了我一个解决方案，我们看一下</p>
<p><a target="_blank" rel="noopener" href="https://www.modb.pro/db/143244">Spark 之访问 Hive 空指针异常</a></p>
<p>因为学长说我的rootcause应该是<code> Caused by: java.lang.NullPointerException</code>，这里还要普及一个小知识点</p>
<blockquote>
<p>看报错的时候要从下网上看，最下面的是最根本的原因</p>
</blockquote>
<p>根据文章所说的添加了winutils.exe就行，具体的原因还是–&gt;<code>Spark需要用到Hadoop中的一些类库&amp;使用winutils.exe文件初始化Hive的上下文</code></p>
<h5 id="table-or-view-not-found"><a href="#table-or-view-not-found" class="headerlink" title="table or view not found"></a>table or view not found</h5><p>如果遇到<code>table or view not found</code>，我是因为sparkSession没有打开hive支持</p>
<h3 id="热词词云"><a href="#热词词云" class="headerlink" title="热词词云"></a>热词词云</h3><h5 id="上传文件"><a href="#上传文件" class="headerlink" title="上传文件"></a>上传文件</h5><p>将文件上传至hdfs中</p>
<pre class="line-numbers language-shell" data-language="shell"><code class="language-shell">hdfs dfs -put &#x2F;data&#x2F;SogouQ.sample.txt &#x2F;data&#x2F;<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<h5 id="类文件"><a href="#类文件" class="headerlink" title="类文件"></a>类文件</h5><pre class="line-numbers language-java" data-language="java"><code class="language-java"><span class="token keyword">package</span> <span class="token namespace">com<span class="token punctuation">.</span>hengxing<span class="token punctuation">.</span>ecommerceuserprofile<span class="token punctuation">.</span>etls<span class="token punctuation">.</span>platform</span><span class="token punctuation">;</span>

<span class="token keyword">import</span> <span class="token namespace">org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span></span><span class="token class-name">SparkConf</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>api<span class="token punctuation">.</span>java<span class="token punctuation">.</span></span><span class="token class-name">JavaPairRDD</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>api<span class="token punctuation">.</span>java<span class="token punctuation">.</span></span><span class="token class-name">JavaRDD</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>api<span class="token punctuation">.</span>java<span class="token punctuation">.</span></span><span class="token class-name">JavaSparkContext</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>api<span class="token punctuation">.</span>java<span class="token punctuation">.</span>function<span class="token punctuation">.</span></span><span class="token class-name">Function2</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>api<span class="token punctuation">.</span>java<span class="token punctuation">.</span>function<span class="token punctuation">.</span></span><span class="token class-name">PairFunction</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">scala<span class="token punctuation">.</span></span><span class="token class-name">Tuple2</span><span class="token punctuation">;</span>

<span class="token keyword">import</span> <span class="token namespace">java<span class="token punctuation">.</span>util<span class="token punctuation">.</span></span><span class="token class-name">List</span><span class="token punctuation">;</span>

<span class="token comment">/**
 * @projectName: EcommerceUserProfile
 * @package: com.hengxing.ecommerceuserprofile.etls.platform
 * @className: HotWordEtl
 * @author: HengxingStu
 * @description: TODO
 * @date: 4/12/2022 11:19 PM
 * @version: 1.0
 */</span>
<span class="token keyword">public</span> <span class="token keyword">class</span> <span class="token class-name">HotWordEtl</span> <span class="token punctuation">&#123;</span>
    <span class="token keyword">public</span> <span class="token keyword">static</span> <span class="token keyword">void</span> <span class="token function">main</span><span class="token punctuation">(</span><span class="token class-name">String</span><span class="token punctuation">[</span><span class="token punctuation">]</span> args<span class="token punctuation">)</span> <span class="token punctuation">&#123;</span>
        <span class="token comment">//创建一个javaspark context ，方便后面调用transform操作</span>
        <span class="token class-name">SparkConf</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">SparkConf</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
                <span class="token punctuation">.</span><span class="token function">setAppName</span><span class="token punctuation">(</span><span class="token string">"Hot word etl"</span><span class="token punctuation">)</span>
                <span class="token punctuation">.</span><span class="token function">setMaster</span><span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token class-name">JavaSparkContext</span> jsc <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">JavaSparkContext</span><span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span><span class="token punctuation">;</span>

        <span class="token comment">//1.先从hdfs读数据</span>
        <span class="token class-name">System</span><span class="token punctuation">.</span><span class="token function">setProperty</span><span class="token punctuation">(</span><span class="token string">"HADOOP_USER_NAME"</span><span class="token punctuation">,</span><span class="token string">"hengxing"</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token class-name">JavaRDD</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">String</span><span class="token punctuation">></span></span> linesRdd <span class="token operator">=</span> jsc<span class="token punctuation">.</span><span class="token function">textFile</span><span class="token punctuation">(</span><span class="token string">"hdfs://hengxing:9000/data/SogouQ.sample.txt"</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

        <span class="token comment">//2.mapToPair得到二元组，准备word count</span>
        <span class="token class-name">JavaPairRDD</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">String</span><span class="token punctuation">,</span> <span class="token class-name">Integer</span><span class="token punctuation">></span></span> pairRDD <span class="token operator">=</span> linesRdd<span class="token punctuation">.</span><span class="token function">mapToPair</span><span class="token punctuation">(</span><span class="token keyword">new</span> <span class="token class-name">PairFunction</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">String</span><span class="token punctuation">,</span> <span class="token class-name">String</span><span class="token punctuation">,</span> <span class="token class-name">Integer</span><span class="token punctuation">></span></span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token punctuation">&#123;</span>
            <span class="token annotation punctuation">@Override</span>
            <span class="token keyword">public</span> <span class="token class-name">Tuple2</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">String</span><span class="token punctuation">,</span> <span class="token class-name">Integer</span><span class="token punctuation">></span></span> <span class="token function">call</span><span class="token punctuation">(</span><span class="token class-name">String</span> s<span class="token punctuation">)</span> <span class="token keyword">throws</span> <span class="token class-name">Exception</span> <span class="token punctuation">&#123;</span>
                <span class="token class-name">String</span> word <span class="token operator">=</span> s<span class="token punctuation">.</span><span class="token function">split</span><span class="token punctuation">(</span><span class="token string">"\t"</span><span class="token punctuation">)</span><span class="token punctuation">[</span><span class="token number">2</span><span class="token punctuation">]</span><span class="token punctuation">;</span>
                <span class="token keyword">return</span> <span class="token keyword">new</span> <span class="token class-name">Tuple2</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token punctuation">></span></span><span class="token punctuation">(</span>word<span class="token punctuation">,</span> <span class="token number">1</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
            <span class="token punctuation">&#125;</span>
        <span class="token punctuation">&#125;</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

        <span class="token comment">//3.以word为key，通过reducebykey聚合</span>
        <span class="token class-name">JavaPairRDD</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">String</span><span class="token punctuation">,</span> <span class="token class-name">Integer</span><span class="token punctuation">></span></span> countRdd <span class="token operator">=</span> pairRDD<span class="token punctuation">.</span><span class="token function">reduceByKey</span><span class="token punctuation">(</span><span class="token keyword">new</span> <span class="token class-name">Function2</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Integer</span><span class="token punctuation">,</span> <span class="token class-name">Integer</span><span class="token punctuation">,</span> <span class="token class-name">Integer</span><span class="token punctuation">></span></span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token punctuation">&#123;</span>
            <span class="token annotation punctuation">@Override</span>
            <span class="token keyword">public</span> <span class="token class-name">Integer</span> <span class="token function">call</span><span class="token punctuation">(</span><span class="token class-name">Integer</span> integer<span class="token punctuation">,</span> <span class="token class-name">Integer</span> integer2<span class="token punctuation">)</span> <span class="token keyword">throws</span> <span class="token class-name">Exception</span> <span class="token punctuation">&#123;</span>
                <span class="token keyword">return</span> integer <span class="token operator">+</span> integer2<span class="token punctuation">;</span>
            <span class="token punctuation">&#125;</span>
        <span class="token punctuation">&#125;</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

        <span class="token comment">//排序，但是由于sortbykey只能根据第一个元素进行排序，所以排序前需要调整参数位置</span>
<span class="token comment">//       1.exchange position first</span>
        <span class="token class-name">JavaPairRDD</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Integer</span><span class="token punctuation">,</span> <span class="token class-name">String</span><span class="token punctuation">></span></span> swapedRdd <span class="token operator">=</span> countRdd<span class="token punctuation">.</span><span class="token function">mapToPair</span><span class="token punctuation">(</span><span class="token keyword">new</span> <span class="token class-name">PairFunction</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Tuple2</span><span class="token punctuation">&lt;</span><span class="token class-name">String</span><span class="token punctuation">,</span> <span class="token class-name">Integer</span><span class="token punctuation">></span><span class="token punctuation">,</span> <span class="token class-name">Integer</span><span class="token punctuation">,</span> <span class="token class-name">String</span><span class="token punctuation">></span></span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token punctuation">&#123;</span>
            <span class="token annotation punctuation">@Override</span>
            <span class="token keyword">public</span> <span class="token class-name">Tuple2</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Integer</span><span class="token punctuation">,</span> <span class="token class-name">String</span><span class="token punctuation">></span></span> <span class="token function">call</span><span class="token punctuation">(</span><span class="token class-name">Tuple2</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">String</span><span class="token punctuation">,</span> <span class="token class-name">Integer</span><span class="token punctuation">></span></span> stringIntegerTuple2<span class="token punctuation">)</span> <span class="token keyword">throws</span> <span class="token class-name">Exception</span> <span class="token punctuation">&#123;</span>
                <span class="token keyword">return</span> stringIntegerTuple2<span class="token punctuation">.</span><span class="token function">swap</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
            <span class="token punctuation">&#125;</span>
        <span class="token punctuation">&#125;</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment">//        2.count desc</span>
        <span class="token class-name">JavaPairRDD</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Integer</span><span class="token punctuation">,</span> <span class="token class-name">String</span><span class="token punctuation">></span></span> sortedRdd <span class="token operator">=</span> swapedRdd<span class="token punctuation">.</span><span class="token function">sortByKey</span><span class="token punctuation">(</span><span class="token boolean">false</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment">//        3.exchange again,back to previous,extract TopN,get a list finally</span>
        <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Tuple2</span><span class="token punctuation">&lt;</span><span class="token class-name">String</span><span class="token punctuation">,</span> <span class="token class-name">Integer</span><span class="token punctuation">></span><span class="token punctuation">></span></span> resultlist <span class="token operator">=</span> sortedRdd<span class="token punctuation">.</span><span class="token function">mapToPair</span><span class="token punctuation">(</span><span class="token keyword">new</span> <span class="token class-name">PairFunction</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Tuple2</span><span class="token punctuation">&lt;</span><span class="token class-name">Integer</span><span class="token punctuation">,</span> <span class="token class-name">String</span><span class="token punctuation">></span><span class="token punctuation">,</span> <span class="token class-name">String</span><span class="token punctuation">,</span> <span class="token class-name">Integer</span><span class="token punctuation">></span></span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token punctuation">&#123;</span>
            <span class="token annotation punctuation">@Override</span>
            <span class="token keyword">public</span> <span class="token class-name">Tuple2</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">String</span><span class="token punctuation">,</span> <span class="token class-name">Integer</span><span class="token punctuation">></span></span> <span class="token function">call</span><span class="token punctuation">(</span><span class="token class-name">Tuple2</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Integer</span><span class="token punctuation">,</span> <span class="token class-name">String</span><span class="token punctuation">></span></span> integerStringTuple2<span class="token punctuation">)</span> <span class="token keyword">throws</span> <span class="token class-name">Exception</span> <span class="token punctuation">&#123;</span>
                <span class="token keyword">return</span> integerStringTuple2<span class="token punctuation">.</span><span class="token function">swap</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
            <span class="token punctuation">&#125;</span>
        <span class="token punctuation">&#125;</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">take</span><span class="token punctuation">(</span><span class="token number">10</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

<span class="token comment">//        print</span>
        <span class="token keyword">for</span> <span class="token punctuation">(</span><span class="token class-name">Tuple2</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">String</span><span class="token punctuation">,</span> <span class="token class-name">Integer</span><span class="token punctuation">></span></span> hotWordCount<span class="token operator">:</span> resultlist<span class="token punctuation">)</span><span class="token punctuation">&#123;</span>
            <span class="token class-name">System</span><span class="token punctuation">.</span>out<span class="token punctuation">.</span><span class="token function">println</span><span class="token punctuation">(</span>hotWordCount<span class="token punctuation">.</span>_1 <span class="token operator">+</span> <span class="token string">"=== count "</span> <span class="token operator">+</span> hotWordCount<span class="token punctuation">.</span>_2 <span class="token operator">+</span> <span class="token string">"\n"</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token punctuation">&#125;</span>
    <span class="token punctuation">&#125;</span>
<span class="token punctuation">&#125;</span>
<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<h3 id="平台近期数据增量"><a href="#平台近期数据增量" class="headerlink" title="平台近期数据增量"></a>平台近期数据增量</h3><p>SQL中的模板字符串<code>&#39;%s&#39;</code>，这样就可以将字符插进去，就像这样</p>
<pre class="line-numbers language-java" data-language="java"><code class="language-java">最后合成的字符串 <span class="token operator">=</span> <span class="token class-name">String</span><span class="token punctuation">.</span><span class="token function">format</span><span class="token punctuation">(</span>你的SQL字符串<span class="token punctuation">,</span> 要添加的字符串<span class="token punctuation">)</span><span class="token punctuation">;</span><span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>GrowthEtl.java</p>
<pre class="line-numbers language-java" data-language="java"><code class="language-java"><span class="token keyword">package</span> <span class="token namespace">com<span class="token punctuation">.</span>hengxing<span class="token punctuation">.</span>ecommerceuserprofile<span class="token punctuation">.</span>etls<span class="token punctuation">.</span>platform</span><span class="token punctuation">;</span>

<span class="token keyword">import</span> <span class="token namespace">com<span class="token punctuation">.</span>alibaba<span class="token punctuation">.</span>fastjson<span class="token punctuation">.</span></span><span class="token class-name">JSONObject</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">com<span class="token punctuation">.</span>hengxing<span class="token punctuation">.</span>ecommerceuserprofile<span class="token punctuation">.</span>utils<span class="token punctuation">.</span></span><span class="token class-name">SparkUtils</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">com<span class="token punctuation">.</span>hengxing<span class="token punctuation">.</span>ecommerceuserprofile<span class="token punctuation">.</span>utils<span class="token punctuation">.</span>dateutils<span class="token punctuation">.</span></span><span class="token class-name">DateStyle</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">com<span class="token punctuation">.</span>hengxing<span class="token punctuation">.</span>ecommerceuserprofile<span class="token punctuation">.</span>utils<span class="token punctuation">.</span>dateutils<span class="token punctuation">.</span></span><span class="token class-name">DateUtil</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">lombok<span class="token punctuation">.</span></span><span class="token class-name">Data</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>sql<span class="token punctuation">.</span></span><span class="token class-name">Dataset</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>sql<span class="token punctuation">.</span></span><span class="token class-name">Row</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>sql<span class="token punctuation">.</span></span><span class="token class-name">SparkSession</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>sql<span class="token punctuation">.</span>types<span class="token punctuation">.</span></span><span class="token class-name">StructType</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">scala<span class="token punctuation">.</span></span><span class="token class-name">Tuple2</span><span class="token punctuation">;</span>

<span class="token keyword">import</span> <span class="token namespace">java<span class="token punctuation">.</span>math<span class="token punctuation">.</span></span><span class="token class-name">BigDecimal</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">java<span class="token punctuation">.</span>time<span class="token punctuation">.</span></span><span class="token class-name">LocalDate</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">java<span class="token punctuation">.</span>time<span class="token punctuation">.</span></span><span class="token class-name">Month</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">java<span class="token punctuation">.</span>time<span class="token punctuation">.</span></span><span class="token class-name">ZoneId</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">java<span class="token punctuation">.</span>util<span class="token punctuation">.</span></span><span class="token class-name">ArrayList</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">java<span class="token punctuation">.</span>util<span class="token punctuation">.</span></span><span class="token class-name">Date</span><span class="token punctuation">;</span>
<span class="token keyword">import</span> <span class="token namespace">java<span class="token punctuation">.</span>util<span class="token punctuation">.</span></span><span class="token class-name">List</span><span class="token punctuation">;</span>

<span class="token keyword">public</span> <span class="token keyword">class</span> <span class="token class-name">GrowthEtl</span> <span class="token punctuation">&#123;</span>
    <span class="token keyword">public</span> <span class="token keyword">static</span> <span class="token keyword">void</span> <span class="token function">main</span><span class="token punctuation">(</span><span class="token class-name">String</span><span class="token punctuation">[</span><span class="token punctuation">]</span> args<span class="token punctuation">)</span> <span class="token punctuation">&#123;</span>
        <span class="token class-name">SparkSession</span> session <span class="token operator">=</span> <span class="token class-name">SparkUtils</span><span class="token punctuation">.</span><span class="token function">initSession</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">GrowthLineVo</span><span class="token punctuation">></span></span> growthLineVo <span class="token operator">=</span> <span class="token function">growthEtl</span><span class="token punctuation">(</span>session<span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token class-name">System</span><span class="token punctuation">.</span>out<span class="token punctuation">.</span><span class="token function">println</span><span class="token punctuation">(</span>growthLineVo<span class="token punctuation">)</span><span class="token punctuation">;</span>
    <span class="token punctuation">&#125;</span>

    <span class="token comment">//自定义VIewObject，统计每日数据</span>
    <span class="token annotation punctuation">@Data</span>
    <span class="token keyword">static</span> <span class="token keyword">class</span> <span class="token class-name">GrowthLineVo</span> <span class="token punctuation">&#123;</span>
        <span class="token comment">// 每天新增注册数、总用户数、总订单数、总流水GMV</span>
        <span class="token keyword">private</span> <span class="token class-name">String</span> day<span class="token punctuation">;</span>  <span class="token comment">//日期</span>
        <span class="token keyword">private</span> <span class="token class-name">Integer</span> regCount<span class="token punctuation">;</span>  <span class="token comment">// 每天新增注册数</span>
        <span class="token keyword">private</span> <span class="token class-name">Integer</span> memberCount<span class="token punctuation">;</span>  <span class="token comment">// 每天平台总用户数</span>
        <span class="token keyword">private</span> <span class="token class-name">Integer</span> orderCount<span class="token punctuation">;</span>  <span class="token comment">// 每天总订单数</span>
        <span class="token keyword">private</span> <span class="token class-name">BigDecimal</span> gmv<span class="token punctuation">;</span>  <span class="token comment">// 每天平台订单总流水金额</span>
    <span class="token punctuation">&#125;</span>

    <span class="token keyword">private</span> <span class="token keyword">static</span> <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">GrowthLineVo</span><span class="token punctuation">></span></span> <span class="token function">growthEtl</span><span class="token punctuation">(</span><span class="token class-name">SparkSession</span> session<span class="token punctuation">)</span> <span class="token punctuation">&#123;</span>
        <span class="token comment">// 指定“当前日期”是2019.11.30，这是数据决定的</span>
        <span class="token class-name">LocalDate</span> now <span class="token operator">=</span> <span class="token class-name">LocalDate</span><span class="token punctuation">.</span><span class="token function">of</span><span class="token punctuation">(</span><span class="token number">2019</span><span class="token punctuation">,</span> <span class="token class-name">Month</span><span class="token punctuation">.</span>NOVEMBER<span class="token punctuation">,</span> <span class="token number">30</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token class-name">Date</span> nowDay <span class="token operator">=</span> <span class="token class-name">Date</span><span class="token punctuation">.</span><span class="token function">from</span><span class="token punctuation">(</span>now<span class="token punctuation">.</span><span class="token function">atStartOfDay</span><span class="token punctuation">(</span><span class="token class-name">ZoneId</span><span class="token punctuation">.</span><span class="token function">systemDefault</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">toInstant</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token class-name">Date</span> sevenDayBefore <span class="token operator">=</span> <span class="token class-name">DateUtil</span><span class="token punctuation">.</span><span class="token function">addDay</span><span class="token punctuation">(</span>nowDay<span class="token punctuation">,</span> <span class="token operator">-</span><span class="token number">7</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

        <span class="token comment">// 近七天注册人数统计（总量、新增）</span>
        <span class="token class-name">String</span> memberSql <span class="token operator">=</span> <span class="token string">"select date_format(create_time,'yyyy-MM-dd') as day,"</span> <span class="token operator">+</span>
                <span class="token string">" count(id) as regCount, max(id) as memberCount "</span> <span class="token operator">+</span>
                <span class="token string">" from ecommerce.t_member where create_time >='%s' "</span> <span class="token operator">+</span>
                <span class="token string">" group by date_format(create_time,'yyyy-MM-dd') order by day"</span><span class="token punctuation">;</span>

        memberSql <span class="token operator">=</span> <span class="token class-name">String</span><span class="token punctuation">.</span><span class="token function">format</span><span class="token punctuation">(</span>memberSql<span class="token punctuation">,</span> <span class="token class-name">DateUtil<span class="token punctuation">.</span>DateToString</span><span class="token punctuation">(</span>sevenDayBefore<span class="token punctuation">,</span> <span class="token class-name">DateStyle</span><span class="token punctuation">.</span>YYYY_MM_DD_HH_MM_SS<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token class-name">Dataset</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Row</span><span class="token punctuation">></span></span> memberDs <span class="token operator">=</span> session<span class="token punctuation">.</span><span class="token function">sql</span><span class="token punctuation">(</span>memberSql<span class="token punctuation">)</span><span class="token punctuation">;</span>

        <span class="token comment">// 近七天订单和流水统计（新增总流水）</span>
        <span class="token class-name">String</span> orderSql <span class="token operator">=</span> <span class="token string">"select date_format(create_time,'yyyy-MM-dd') as day,"</span> <span class="token operator">+</span>
                <span class="token string">" max(order_id) orderCount, sum(origin_price) as gmv"</span> <span class="token operator">+</span>
                <span class="token string">" from ecommerce.t_order where create_time >='%s' "</span> <span class="token operator">+</span>
                <span class="token string">"group by date_format(create_time,'yyyy-MM-dd') order by day"</span><span class="token punctuation">;</span>

        orderSql <span class="token operator">=</span> <span class="token class-name">String</span><span class="token punctuation">.</span><span class="token function">format</span><span class="token punctuation">(</span>orderSql<span class="token punctuation">,</span> <span class="token class-name">DateUtil<span class="token punctuation">.</span>DateToString</span><span class="token punctuation">(</span>sevenDayBefore<span class="token punctuation">,</span> <span class="token class-name">DateStyle</span><span class="token punctuation">.</span>YYYY_MM_DD_HH_MM_SS<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token class-name">Dataset</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Row</span><span class="token punctuation">></span></span> orderDs <span class="token operator">=</span> session<span class="token punctuation">.</span><span class="token function">sql</span><span class="token punctuation">(</span>orderSql<span class="token punctuation">)</span><span class="token punctuation">;</span>

        <span class="token comment">// 联接两个查询结果，按照day进行内连接</span>
        <span class="token class-name">Dataset</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Tuple2</span><span class="token punctuation">&lt;</span><span class="token class-name">Row</span><span class="token punctuation">,</span> <span class="token class-name">Row</span><span class="token punctuation">></span><span class="token punctuation">></span></span> tuple2Dataset <span class="token operator">=</span> memberDs<span class="token punctuation">.</span><span class="token function">joinWith</span><span class="token punctuation">(</span>orderDs<span class="token punctuation">,</span> memberDs<span class="token punctuation">.</span><span class="token function">col</span><span class="token punctuation">(</span><span class="token string">"day"</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">equalTo</span><span class="token punctuation">(</span>orderDs<span class="token punctuation">.</span><span class="token function">col</span><span class="token punctuation">(</span><span class="token string">"day"</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">,</span> <span class="token string">"inner"</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

        <span class="token comment">//先转换成list，然后遍历，取出每一天的数据，转成VO，放入新的list</span>
        <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Tuple2</span><span class="token punctuation">&lt;</span><span class="token class-name">Row</span><span class="token punctuation">,</span> <span class="token class-name">Row</span><span class="token punctuation">></span><span class="token punctuation">></span></span> tuple2s <span class="token operator">=</span> tuple2Dataset<span class="token punctuation">.</span><span class="token function">collectAsList</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">GrowthLineVo</span><span class="token punctuation">></span></span> vos <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">ArrayList</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token punctuation">></span></span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

        <span class="token comment">// 遍历二元组List，包装 GrowthLineVo</span>
        <span class="token keyword">for</span> <span class="token punctuation">(</span><span class="token class-name">Tuple2</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Row</span><span class="token punctuation">,</span> <span class="token class-name">Row</span><span class="token punctuation">></span></span> tuple2 <span class="token operator">:</span> tuple2s<span class="token punctuation">)</span> <span class="token punctuation">&#123;</span>
            <span class="token class-name">Row</span> row1 <span class="token operator">=</span> tuple2<span class="token punctuation">.</span><span class="token function">_1</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>    <span class="token comment">// memberSql结果</span>
            <span class="token class-name">Row</span> row2 <span class="token operator">=</span> tuple2<span class="token punctuation">.</span><span class="token function">_2</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>    <span class="token comment">// orderSql结果</span>

            <span class="token class-name">JSONObject</span> obj <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">JSONObject</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

            <span class="token comment">//提取row类型里的所有字段</span>
            <span class="token class-name">StructType</span> schema <span class="token operator">=</span> row1<span class="token punctuation">.</span><span class="token function">schema</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
            <span class="token class-name">String</span><span class="token punctuation">[</span><span class="token punctuation">]</span> strings <span class="token operator">=</span> schema<span class="token punctuation">.</span><span class="token function">fieldNames</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
            <span class="token keyword">for</span> <span class="token punctuation">(</span><span class="token class-name">String</span> string <span class="token operator">:</span> strings<span class="token punctuation">)</span> <span class="token punctuation">&#123;</span>
                <span class="token class-name">Object</span> as <span class="token operator">=</span> row1<span class="token punctuation">.</span><span class="token function">getAs</span><span class="token punctuation">(</span>string<span class="token punctuation">)</span><span class="token punctuation">;</span>
                obj<span class="token punctuation">.</span><span class="token function">put</span><span class="token punctuation">(</span>string<span class="token punctuation">,</span> as<span class="token punctuation">)</span><span class="token punctuation">;</span>
            <span class="token punctuation">&#125;</span>

            schema <span class="token operator">=</span> row2<span class="token punctuation">.</span><span class="token function">schema</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
            strings <span class="token operator">=</span> schema<span class="token punctuation">.</span><span class="token function">fieldNames</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
            <span class="token keyword">for</span> <span class="token punctuation">(</span><span class="token class-name">String</span> string <span class="token operator">:</span> strings<span class="token punctuation">)</span> <span class="token punctuation">&#123;</span>
                <span class="token class-name">Object</span> as <span class="token operator">=</span> row2<span class="token punctuation">.</span><span class="token function">getAs</span><span class="token punctuation">(</span>string<span class="token punctuation">)</span><span class="token punctuation">;</span>
                obj<span class="token punctuation">.</span><span class="token function">put</span><span class="token punctuation">(</span>string<span class="token punctuation">,</span> as<span class="token punctuation">)</span><span class="token punctuation">;</span>
            <span class="token punctuation">&#125;</span>

            <span class="token class-name">GrowthLineVo</span> growthLineVo <span class="token operator">=</span> obj<span class="token punctuation">.</span><span class="token function">toJavaObject</span><span class="token punctuation">(</span><span class="token class-name">GrowthLineVo</span><span class="token punctuation">.</span><span class="token keyword">class</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
            vos<span class="token punctuation">.</span><span class="token function">add</span><span class="token punctuation">(</span>growthLineVo<span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token punctuation">&#125;</span>

        <span class="token comment">// 七天前，再之前的订单流水总和（GMV）</span>
        <span class="token class-name">String</span> preGmvSql <span class="token operator">=</span> <span class="token string">"select sum(origin_price) as totalGmv from ecommerce.t_order where create_time &lt;'%s'"</span><span class="token punctuation">;</span>
        preGmvSql <span class="token operator">=</span> <span class="token class-name">String</span><span class="token punctuation">.</span><span class="token function">format</span><span class="token punctuation">(</span>preGmvSql<span class="token punctuation">,</span> <span class="token class-name">DateUtil<span class="token punctuation">.</span>DateToString</span><span class="token punctuation">(</span>sevenDayBefore<span class="token punctuation">,</span> <span class="token class-name">DateStyle</span><span class="token punctuation">.</span>YYYY_MM_DD_HH_MM_SS<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token class-name">Dataset</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Row</span><span class="token punctuation">></span></span> gmvDs <span class="token operator">=</span> session<span class="token punctuation">.</span><span class="token function">sql</span><span class="token punctuation">(</span>preGmvSql<span class="token punctuation">)</span><span class="token punctuation">;</span>

        <span class="token keyword">double</span> previousGmv <span class="token operator">=</span> gmvDs<span class="token punctuation">.</span><span class="token function">collectAsList</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">get</span><span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">getDouble</span><span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span><span class="token punctuation">;</span><span class="token comment">//取出第0行，第0列</span>
        <span class="token class-name">BigDecimal</span> preGmv <span class="token operator">=</span> <span class="token class-name">BigDecimal</span><span class="token punctuation">.</span><span class="token function">valueOf</span><span class="token punctuation">(</span>previousGmv<span class="token punctuation">)</span><span class="token punctuation">;</span>


        <span class="token comment">// 之前每天的增量gmv取出，依次叠加，得到总和</span>
        <span class="token class-name">BigDecimal</span> currentGmv <span class="token operator">=</span> preGmv<span class="token punctuation">;</span>

        <span class="token keyword">for</span> <span class="token punctuation">(</span><span class="token keyword">int</span> i <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">;</span> i <span class="token operator">&lt;</span> vos<span class="token punctuation">.</span><span class="token function">size</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span> i<span class="token operator">++</span><span class="token punctuation">)</span> <span class="token punctuation">&#123;</span>
            <span class="token class-name">GrowthLineVo</span> growthLineVo <span class="token operator">=</span> vos<span class="token punctuation">.</span><span class="token function">get</span><span class="token punctuation">(</span>i<span class="token punctuation">)</span><span class="token punctuation">;</span>
            currentGmv <span class="token operator">=</span> currentGmv<span class="token punctuation">.</span><span class="token function">add</span><span class="token punctuation">(</span>growthLineVo<span class="token punctuation">.</span><span class="token function">getGmv</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span><span class="token comment">//加上当天GMV新增量</span>

            growthLineVo<span class="token punctuation">.</span><span class="token function">setGmv</span><span class="token punctuation">(</span>currentGmv<span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token punctuation">&#125;</span>
        <span class="token keyword">return</span> vos<span class="token punctuation">;</span>
    <span class="token punctuation">&#125;</span>
<span class="token punctuation">&#125;</span>
<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<h3 id="标签系统"><a href="#标签系统" class="headerlink" title="标签系统"></a>标签系统</h3><p>首先保证es、kibana安装完成，然后在kibana命令行中写如下命令：</p>
<pre class="line-numbers language-json" data-language="json"><code class="language-json">PUT /tag/_doc/_mapping?pretty 
<span class="token punctuation">&#123;</span> 
<span class="token property">"_doc"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> 
<span class="token property">"properties"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> 
<span class="token property">"memberId"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"text"</span><span class="token punctuation">,</span> <span class="token property">"fields"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"keyword"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"keyword"</span> <span class="token punctuation">&#125;</span> <span class="token punctuation">&#125;</span> <span class="token punctuation">&#125;</span><span class="token punctuation">,</span> 
<span class="token property">"phone"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"text"</span><span class="token punctuation">,</span> <span class="token property">"fields"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"keyword"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"keyword"</span> <span class="token punctuation">&#125;</span> <span class="token punctuation">&#125;</span> <span class="token punctuation">&#125;</span><span class="token punctuation">,</span> 
<span class="token property">"sex"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"text"</span><span class="token punctuation">,</span> <span class="token property">"fields"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"keyword"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"keyword"</span> <span class="token punctuation">&#125;</span> <span class="token punctuation">&#125;</span> <span class="token punctuation">&#125;</span><span class="token punctuation">,</span> 
<span class="token property">"channel"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"text"</span><span class="token punctuation">,</span> <span class="token property">"fields"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"keyword"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"keyword"</span> <span class="token punctuation">&#125;</span> <span class="token punctuation">&#125;</span> <span class="token punctuation">&#125;</span><span class="token punctuation">,</span> 
<span class="token property">"subOpenId"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"text"</span><span class="token punctuation">,</span> <span class="token property">"fields"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"keyword"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"keyword"</span> <span class="token punctuation">&#125;</span> <span class="token punctuation">&#125;</span> <span class="token punctuation">&#125;</span><span class="token punctuation">,</span>
“address”<span class="token operator">:</span> <span class="token punctuation">&#123;</span> “type”<span class="token operator">:</span> “text” <span class="token punctuation">&#125;</span><span class="token punctuation">,</span>
<span class="token property">"regTime"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"date"</span> <span class="token punctuation">&#125;</span><span class="token punctuation">,</span> 

<span class="token property">"orderCount"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"long"</span> <span class="token punctuation">&#125;</span><span class="token punctuation">,</span> 
<span class="token property">"orderTime"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"date"</span> <span class="token punctuation">&#125;</span><span class="token punctuation">,</span> 
<span class="token property">"orderMoney"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"float"</span> <span class="token punctuation">&#125;</span><span class="token punctuation">,</span> 
<span class="token property">"favGoods"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"text"</span><span class="token punctuation">,</span> <span class="token property">"fields"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"keyword"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"keyword"</span> <span class="token punctuation">&#125;</span> <span class="token punctuation">&#125;</span> <span class="token punctuation">&#125;</span><span class="token punctuation">,</span> 

<span class="token property">"freeCouponTime"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"date"</span> <span class="token punctuation">&#125;</span><span class="token punctuation">,</span> 
<span class="token property">"couponTimes"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"date"</span> <span class="token punctuation">&#125;</span><span class="token punctuation">,</span> 
<span class="token property">"chargeMoney"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"double"</span> <span class="token punctuation">&#125;</span><span class="token punctuation">,</span> 

<span class="token property">"overTime"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"long"</span> <span class="token punctuation">&#125;</span><span class="token punctuation">,</span> 
<span class="token property">"feedBack"</span><span class="token operator">:</span> <span class="token punctuation">&#123;</span> <span class="token property">"type"</span><span class="token operator">:</span> <span class="token string">"long"</span> <span class="token punctuation">&#125;</span>
<span class="token punctuation">&#125;</span> 
<span class="token punctuation">&#125;</span> 
<span class="token punctuation">&#125;</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>正常情况返回</p>
<pre class="line-numbers language-json" data-language="json"><code class="language-json"><span class="token punctuation">&#123;</span>
  <span class="token property">"acknowledged"</span> <span class="token operator">:</span> <span class="token boolean">true</span>
<span class="token punctuation">&#125;</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<p>也可以在dashboard-&gt;Index Management-&gt;选择你的查询index并点击-&gt;mapping，可以查看到提交的mapping</p>
<p>执行UserTagEtl.java可以将用户标签数据存入es，接着可以在es中查到数据<code>GET /usertag/_search</code>，其中的total表示有1000条数据</p>
<pre class="line-numbers language-json" data-language="json"><code class="language-json"><span class="token property">"hits"</span> <span class="token operator">:</span> <span class="token punctuation">&#123;</span>
    <span class="token property">"total"</span> <span class="token operator">:</span> <span class="token number">1000</span><span class="token punctuation">,</span>
    <span class="token property">"max_score"</span> <span class="token operator">:</span> <span class="token number">1.0</span><span class="token punctuation">,</span>
    <span class="token property">"hits"</span> <span class="token operator">:</span> <span class="token punctuation">[</span>----省略一万字-----<span class="token punctuation">]</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span></span></code></pre>

</article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta">文章作者: </span><span class="post-copyright-info"><a href="mailto:undefined">hengxingstu</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta">文章链接: </span><span class="post-copyright-info"><a href="https://hengxingstu.gitee.io/2022/04/24/ITguiguUserProfile/">https://hengxingstu.gitee.io/2022/04/24/ITguiguUserProfile/</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta">版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来自 <a href="https://hengxingstu.gitee.io" target="_blank">恒星同学</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/tags/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></div><div class="post_share"><div class="social-share" data-image="https://w.wallhaven.cc/full/e7/wallhaven-e7g71w.jpg" data-sites="facebook,twitter,wechat,weibo,qq"></div><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/social-share.js/dist/css/share.min.css" media="print" onload="this.media='all'"><script src="https://cdn.jsdelivr.net/npm/social-share.js/dist/js/social-share.min.js" defer></script></div></div><nav class="pagination-post" id="pagination"><div class="prev-post pull-left"><a href="/2022/04/26/%5BYoutube%5DSpringBoot+Angular%E5%AE%9E%E6%88%98/"><img class="prev-cover" src="https://wallpapercave.com/uwp/uwp2365506.jpeg" onerror="onerror=null;src='/img/404.jpg'" alt="cover of previous post"><div class="pagination-info"><div class="label">上一篇</div><div class="prev_info">Youtube_SpringBoot+Angular实战</div></div></a></div><div class="next-post pull-right"><a href="/2022/04/24/webdata%E6%95%B0%E6%8D%AE/"><img class="next-cover" src="https://wallpapercave.com/uwp/uwp2361079.png" onerror="onerror=null;src='/img/404.jpg'" alt="cover of next post"><div class="pagination-info"><div class="label">下一篇</div><div class="next_info">webData用户数据【JSON格式】</div></div></a></div></nav><div class="relatedPosts"><div class="headline"><i class="fas fa-thumbs-up fa-fw"></i><span>相关推荐</span></div><div class="relatedPosts-list"><div><a href="/2022/04/24/webdata%E6%95%B0%E6%8D%AE/" title="webData用户数据【JSON格式】"><img class="cover" src="https://wallpapercave.com/uwp/uwp2361079.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2022-04-24</div><div class="title">webData用户数据【JSON格式】</div></div></a></div></div></div><hr/><div id="post-comment"><div class="comment-head"><div class="comment-headline"><i class="fas fa-comments fa-fw"></i><span> 评论</span></div></div><div class="comment-wrap"><div><div id="twikoo-wrap"></div></div></div></div></div><div class="aside-content" id="aside-content"><div class="card-widget card-info"><div class="is-center"><div class="avatar-img"><img src="https://wallpapercave.com/uwp/uwp2365506.jpeg" onerror="this.onerror=null;this.src='/img/friend_404.gif'" alt="avatar"/></div><div class="author-info__name">hengxingstu</div><div class="author-info__description"></div></div><div class="card-info-data is-center"><div class="card-info-data-item"><a href="/archives/"><div class="headline">文章</div><div class="length-num">36</div></a></div><div class="card-info-data-item"><a href="/tags/"><div class="headline">标签</div><div class="length-num">23</div></a></div><div class="card-info-data-item"><a href="/categories/"><div class="headline">分类</div><div class="length-num">8</div></a></div></div><a id="card-info-btn" target="_blank" rel="noopener" href="https://github.com/hengxingstu"><i class="fab fa-github"></i><span>Follow Me</span></a><div class="card-info-social-icons is-center"><a class="social-icon" href="https://github.com/hengxingstu" target="_blank" title="Github"><i class="fab fa-github"></i></a><a class="social-icon" href="mailto:hengxingstu@gmail.com" target="_blank" title="Email"><i class="fas fa-envelope"></i></a></div></div><div class="card-widget card-announcement"><div class="item-headline"><i class="fas fa-bullhorn fa-shake"></i><span>公告</span></div><div class="announcement_content">这是我的博客，欢迎和我联系，交流是最好的老师</div></div><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span><span class="toc-percentage"></span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#%E7%8E%AF%E5%A2%83%E9%85%8D%E7%BD%AE"><span class="toc-text">环境配置</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E7%B3%BB%E7%BB%9F%E5%87%86%E5%A4%87"><span class="toc-text">系统准备</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Hadoop%E5%AE%89%E8%A3%85"><span class="toc-text">Hadoop安装</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#Hadoop-%E8%B7%91MapReduce-Job-%E7%9A%84%E6%97%B6%E5%80%99%E5%8D%A1%E4%BD%8F-unhealthy-nodes-x3D-1"><span class="toc-text">Hadoop 跑MapReduce Job 的时候卡住,unhealthy-nodes&#x3D;1</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Hive%E5%AE%89%E8%A3%85"><span class="toc-text">Hive安装</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%AE%89%E8%A3%85"><span class="toc-text">安装</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%90%AF%E5%8A%A8hive%E5%85%83%E5%AD%98%E5%82%A8thrift%E6%9C%8D%E5%8A%A1%E5%99%A8"><span class="toc-text">启动hive元存储thrift服务器</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#HIVE%E6%97%A0%E6%B3%95%E6%AD%A3%E5%B8%B8%E5%90%AF%E5%8A%A8%EF%BC%8C%E6%8A%A5%E6%89%BE%E4%B8%8D%E5%88%B0jdbc%E7%9A%84%E9%A9%B1%E5%8A%A8%E7%B1%BB"><span class="toc-text">HIVE无法正常启动，报找不到jdbc的驱动类</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%AF%BC%E5%85%A5%E6%95%B0%E6%8D%AE"><span class="toc-text">导入数据</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E8%A1%A8%E8%AE%BE%E8%AE%A1"><span class="toc-text">表设计</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E9%85%8D%E7%BD%AEsqoop%E5%81%9A%E6%95%B0%E6%8D%AE%E8%BF%81%E7%A7%BB"><span class="toc-text">配置sqoop做数据迁移</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#yarn-%E6%9D%80%E6%AD%BB-application-%E5%91%BD%E4%BB%A4"><span class="toc-text">yarn 杀死 application 命令</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#hive%E6%B2%A1%E6%9C%89%E6%89%BE%E5%88%B0%E7%B1%BBHiveConf"><span class="toc-text">hive没有找到类HiveConf</span></a></li></ol></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#ES%E5%AE%89%E8%A3%85"><span class="toc-text">ES安装</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E4%B8%8B%E8%BD%BD%E3%80%81%E5%AE%89%E8%A3%85%E3%80%81%E9%85%8D%E7%BD%AE"><span class="toc-text">下载、安装、配置</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%90%AF%E5%8A%A8%E5%91%BD%E4%BB%A4"><span class="toc-text">启动命令</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%8A%A5%E9%94%99"><span class="toc-text">报错</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#kibana%E5%AE%89%E8%A3%85"><span class="toc-text">kibana安装</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E4%B8%8B%E8%BD%BD%E3%80%81%E5%AE%89%E8%A3%85%E3%80%81%E9%85%8D%E7%BD%AE-1"><span class="toc-text">下载、安装、配置</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%A6%82%E4%BD%95%E4%BD%BF%E7%94%A8"><span class="toc-text">如何使用</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%AE%9E%E4%BE%8B"><span class="toc-text">实例</span></a></li></ol></li></ol></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E9%A1%B9%E7%9B%AE%E6%9E%B6%E6%9E%84%E5%92%8C%E5%B7%A5%E5%85%B7%E7%B1%BB"><span class="toc-text">项目架构和工具类</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#IDEA%E9%A1%B9%E7%9B%AE"><span class="toc-text">IDEA项目</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E7%BC%96%E5%86%99%E7%B1%BB%E6%96%87%E4%BB%B6"><span class="toc-text">编写类文件</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E4%B8%AA%E4%BA%BA%E6%A0%87%E7%AD%BE%E5%B1%95%E7%A4%BA"><span class="toc-text">个人标签展示</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E6%8A%A5%E9%94%99-1"><span class="toc-text">报错</span></a><ol class="toc-child"><li class="toc-item toc-level-5"><a class="toc-link" href="#Caused-by-java-lang-NullPointerException"><span class="toc-text">Caused by: java.lang.NullPointerException</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#table-or-view-not-found"><span class="toc-text">table or view not found</span></a></li></ol></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E7%83%AD%E8%AF%8D%E8%AF%8D%E4%BA%91"><span class="toc-text">热词词云</span></a><ol class="toc-child"><li class="toc-item toc-level-5"><a class="toc-link" href="#%E4%B8%8A%E4%BC%A0%E6%96%87%E4%BB%B6"><span class="toc-text">上传文件</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#%E7%B1%BB%E6%96%87%E4%BB%B6"><span class="toc-text">类文件</span></a></li></ol></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%B9%B3%E5%8F%B0%E8%BF%91%E6%9C%9F%E6%95%B0%E6%8D%AE%E5%A2%9E%E9%87%8F"><span class="toc-text">平台近期数据增量</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%A0%87%E7%AD%BE%E7%B3%BB%E7%BB%9F"><span class="toc-text">标签系统</span></a></li></ol></li></ol></li></ol></div></div><div class="card-widget card-recent-post"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item"><a class="thumbnail" href="/2023/02/21/code/springMVC%E6%96%87%E6%A1%A3/" title="Spring参考文档"><img src="https://s2.loli.net/2023/02/21/vdVhPU5RITfrXg4.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Spring参考文档"/></a><div class="content"><a class="title" href="/2023/02/21/code/springMVC%E6%96%87%E6%A1%A3/" title="Spring参考文档">Spring参考文档</a><time datetime="2023-02-21T12:26:00.000Z" title="发表于 2023-02-21 20:26:00">2023-02-21</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2023/02/16/code/SpringMVC%E7%AC%94%E8%AE%B0/" title="SpringMVC学习笔记"><img src="https://s2.loli.net/2023/02/16/N8E1LGTUnYVdmPk.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="SpringMVC学习笔记"/></a><div class="content"><a class="title" href="/2023/02/16/code/SpringMVC%E7%AC%94%E8%AE%B0/" title="SpringMVC学习笔记">SpringMVC学习笔记</a><time datetime="2023-02-16T09:06:00.000Z" title="发表于 2023-02-16 17:06:00">2023-02-16</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2023/02/16/code/Spring%E4%BA%8B%E5%8A%A1/" title="Spring学习笔记"><img src="https://s2.loli.net/2023/02/16/N8E1LGTUnYVdmPk.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Spring学习笔记"/></a><div class="content"><a class="title" href="/2023/02/16/code/Spring%E4%BA%8B%E5%8A%A1/" title="Spring学习笔记">Spring学习笔记</a><time datetime="2023-02-16T09:06:00.000Z" title="发表于 2023-02-16 17:06:00">2023-02-16</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2023/02/15/code/SpringAop%E7%AC%94%E8%AE%B0/" title="SpringAOP笔记"><img src="https://s2.loli.net/2023/02/08/jWNelFA4c7BGkTw.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="SpringAOP笔记"/></a><div class="content"><a class="title" href="/2023/02/15/code/SpringAop%E7%AC%94%E8%AE%B0/" title="SpringAOP笔记">SpringAOP笔记</a><time datetime="2023-02-15T13:00:00.000Z" title="发表于 2023-02-15 21:00:00">2023-02-15</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2023/02/12/code/Spring%E7%AC%94%E8%AE%B0/" title="Spring学习笔记"><img src="https://s2.loli.net/2023/02/12/ukqiYOMfcU4WwCJ.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Spring学习笔记"/></a><div class="content"><a class="title" href="/2023/02/12/code/Spring%E7%AC%94%E8%AE%B0/" title="Spring学习笔记">Spring学习笔记</a><time datetime="2023-02-12T05:58:00.000Z" title="发表于 2023-02-12 13:58:00">2023-02-12</time></div></div></div></div></div></div></main><footer id="footer"><div id="footer-wrap"><div class="copyright">&copy;2020 - 2023 By hengxingstu</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><a id="to_comment" href="#post-comment" title="直达评论"><i class="fas fa-comments"></i></a><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div id="local-search"><div class="search-dialog"><nav class="search-nav"><span class="search-dialog-title">本地搜索</span><span id="loading-status"></span><button class="search-close-button"><i class="fas fa-times"></i></button></nav><div class="is-center" id="loading-database"><i class="fas fa-spinner fa-pulse"></i><span>  数据库加载中</span></div><div class="search-wrap"><div id="local-search-input"><div class="local-search-box"><input class="local-search-box--input" placeholder="搜索文章" type="text"/></div></div><hr/><div id="local-search-results"></div></div></div><div id="search-mask"></div></div><div><script src="/js/utils.js"></script><script src="/js/main.js"></script><script src="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox.umd.js"></script><script src="/js/search/local-search.js"></script><div class="js-pjax"><script>(()=>{
  const init = () => {
    twikoo.init(Object.assign({
      el: '#twikoo-wrap',
      envId: 'https://twikoo-hengxing.vercel.app/',
      region: '',
      onCommentLoaded: function () {
        btf.loadLightbox(document.querySelectorAll('#twikoo .tk-content img:not(.vemoji)'))
      }
    }, null))
  }

  const getCount = () => {
    twikoo.getCommentsCount({
      envId: 'https://twikoo-hengxing.vercel.app/',
      region: '',
      urls: [window.location.pathname],
      includeReply: false
    }).then(function (res) {
      document.getElementById('twikoo-count').innerText = res[0].count
    }).catch(function (err) {
      console.error(err);
    });
  }

  const runFn = () => {
    init()
    GLOBAL_CONFIG_SITE.isPost && getCount()
  }

  const loadTwikoo = () => {
    if (typeof twikoo === 'object') {
      setTimeout(runFn,0)
      return
    } 
    getScript('https://cdn.jsdelivr.net/npm/twikoo/dist/twikoo.all.min.js').then(runFn)
  }

  if ('Twikoo' === 'Twikoo' || !false) {
    if (false) btf.loadComment(document.getElementById('twikoo-wrap'), loadTwikoo)
    else loadTwikoo()
  } else {
    window.loadOtherComment = () => {
      loadTwikoo()
    }
  }
})()</script></div><script defer="defer" id="fluttering_ribbon" mobile="false" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/canvas-fluttering-ribbon.min.js"></script><script src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/activate-power-mode.min.js"></script><script>POWERMODE.colorful = true;
POWERMODE.shake = false;
POWERMODE.mobile = true;
document.body.addEventListener('input', POWERMODE);
</script><script id="click-heart" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/click-heart.min.js" async="async" mobile="true"></script><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div><script src="/live2dw/lib/L2Dwidget.min.js?094cbace49a39548bed64abff5988b05"></script><script>L2Dwidget.init({"pluginRootPath":"live2dw/","pluginJsPath":"lib/","pluginModelPath":"assets/","tagMode":false,"debug":false,"model":{"scale":1,"hHeadPos":0.5,"vHeadPos":0.618,"jsonPath":"/live2dw/assets/haru02.model.json"},"display":{"superSample":2,"width":200,"height":400,"position":"right","hOffset":10,"vOffset":0},"mobile":{"show":false,"scale":0.5},"react":{"opacityDefault":0.7,"opacityOnHover":0.2},"dialog":{"enable":true,"hitokoto":true},"log":false});</script></body></html>